From 3a2ce1d47d1d111c5afe9f7d1bfdd482ff256207 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 17:50:32 -0300 Subject: [PATCH 001/164] =?UTF-8?q?Configurar=20Blazer=20para=20mostrar=20?= =?UTF-8?q?estad=C3=ADsticas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/application_controller.rb | 9 +++++++++ config/blazer.yml | 2 +- config/routes.rb | 7 +++---- db/migrate/20200206163257_install_blazer.rb | 2 -- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index acd0134d..f95159f4 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -62,6 +62,15 @@ class ApplicationController < ActionController::Base render 'application/page_not_found', status: :not_found end + # Necesario para poder acceder a Blazer. Solo les usuaries de este + # sitio pueden acceder al panel. + def require_usuarie + unless find_site.usuarie? current_usuarie + redirect_to root_path + return + end + end + protected def configure_permitted_parameters diff --git a/config/blazer.yml b/config/blazer.yml index 2ba0965f..11792ff1 100644 --- a/config/blazer.yml +++ b/config/blazer.yml @@ -50,7 +50,7 @@ user_method: current_usuarie user_name: email # custom before_action to use for auth -# before_action_method: require_admin +before_action_method: require_usuarie # email to send checks from from_email: blazer@<%= ENV.fetch('SUTTY', 'sutty.nl') %> diff --git a/config/routes.rb b/config/routes.rb index 2c5f1c60..186dd66f 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -4,8 +4,6 @@ Rails.application.routes.draw do devise_for :usuaries get '/.well-known/change-password', to: redirect('/usuaries/edit') - mount Blazer::Engine, at: 'blazer' - root 'application#index' constraints(Constraints::ApiSubdomain.new) do @@ -38,6 +36,9 @@ Rails.application.routes.draw do match '/api/v3/projects/:site_id/notices' => 'api/v1/notices#create', via: %i[post] resources :sites, constraints: { site_id: %r{[^/]+}, id: %r{[^/]+} } do + # Usar Blazer para mostrar estadísticas + mount Blazer::Engine, at: 'stats', as: 'stats' + # Gestionar actualizaciones del sitio get 'pull', to: 'sites#fetch' post 'pull', to: 'sites#merge' @@ -73,7 +74,5 @@ Rails.application.routes.draw do # Compilar el sitio post 'enqueue', to: 'sites#enqueue' post 'reorder_posts', to: 'sites#reorder_posts' - - resources :stats, only: [:index] end end diff --git a/db/migrate/20200206163257_install_blazer.rb b/db/migrate/20200206163257_install_blazer.rb index 9b084169..32799053 100644 --- a/db/migrate/20200206163257_install_blazer.rb +++ b/db/migrate/20200206163257_install_blazer.rb @@ -3,8 +3,6 @@ # Blazer class InstallBlazer < ActiveRecord::Migration[6.0] def change - return unless Rails.env.production? - create_table :blazer_queries do |t| t.references :creator t.string :name From 7511afbf88575fdc62265d0adfe02b56ac764eb6 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 17:51:35 -0300 Subject: [PATCH 002/164] Les usuaries tienen consultas --- app/models/usuarie.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/models/usuarie.rb b/app/models/usuarie.rb index 6de7ba4b..c88dcc68 100644 --- a/app/models/usuarie.rb +++ b/app/models/usuarie.rb @@ -11,6 +11,8 @@ class Usuarie < ApplicationRecord has_many :roles has_many :sites, through: :roles + has_many :blazer_audits, foreign_key: 'user_id', class_name: 'Blazer::Audit' + has_many :blazer_queries, foreign_key: 'creator_id', class_name: 'Blazer::Query' def name email.split('@', 2).first From ddc459130a45b9aa15a590bfa276218facb99030 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 17:54:27 -0300 Subject: [PATCH 003/164] Acceder a la lista de consultas disponibles con la identidad de Sutty --- app/controllers/application_controller.rb | 7 +++++++ app/views/blazer/queries/home.haml | 9 +++++++++ app/views/layouts/_breadcrumb.haml | 2 +- app/views/layouts/blazer/application.haml | 14 ++++++++++++++ config/locales/en.yml | 6 ++++++ config/locales/es.yml | 6 ++++++ 6 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 app/views/blazer/queries/home.haml create mode 100644 app/views/layouts/blazer/application.haml diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index f95159f4..2b5088f7 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -69,6 +69,13 @@ class ApplicationController < ActionController::Base redirect_to root_path return end + + # Necesario para los breadcrumbs. + ActionView::Base.include Loaf::ViewExtensions unless ActionView::Base.included_modules.include? Loaf::ViewExtensions + + breadcrumb current_usuarie.email, main_app.edit_usuarie_registration_path + breadcrumb 'sites.index', main_app.sites_path, match: :exact + breadcrumb 'stats.index', root_path, match: :exact end protected diff --git a/app/views/blazer/queries/home.haml b/app/views/blazer/queries/home.haml new file mode 100644 index 00000000..325ba121 --- /dev/null +++ b/app/views/blazer/queries/home.haml @@ -0,0 +1,9 @@ +#queries + %table.table + %tbody.list + - @queries.each do |query| + %tr + -# + Por alguna razón no tenemos acceso a query_path para poder + generar la URL según Rails + %td= link_to query[:name], "/sites/#{params[:site_id]}/stats/queries/#{query[:to_param]}" diff --git a/app/views/layouts/_breadcrumb.haml b/app/views/layouts/_breadcrumb.haml index c4920bc7..f209787c 100644 --- a/app/views/layouts/_breadcrumb.haml +++ b/app/views/layouts/_breadcrumb.haml @@ -20,5 +20,5 @@ role: 'button', class: 'btn' %li.nav-item - = link_to t('.logout'), destroy_usuarie_session_path, + = link_to t('.logout'), main_app.destroy_usuarie_session_path, method: :delete, role: 'button', class: 'btn' diff --git a/app/views/layouts/blazer/application.haml b/app/views/layouts/blazer/application.haml new file mode 100644 index 00000000..add94190 --- /dev/null +++ b/app/views/layouts/blazer/application.haml @@ -0,0 +1,14 @@ +!!! +%html + %head + %meta{content: 'text/html; charset=UTF-8', 'http-equiv': 'Content-Type'}/ + %title= blazer_title ? blazer_title : 'Sutty' + %meta{charset: 'utf-8'}/ + = favicon_link_tag 'blazer/favicon.png' + = stylesheet_link_tag 'application' + = javascript_pack_tag 'blazer', 'data-turbolinks-track': 'reload' + = csrf_meta_tags + %body{ class: yield(:body) } + .container-fluid#sutty + = render 'layouts/breadcrumb' + = yield diff --git a/config/locales/en.yml b/config/locales/en.yml index fc194eab..7fd83f63 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -576,3 +576,9 @@ en: edit: 'Editing' usuaries: index: 'Users' + stats: + index: 'Statistics' + blazer: + queries: + show: + empty: '(empty)' diff --git a/config/locales/es.yml b/config/locales/es.yml index e8185391..0a48742a 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -584,3 +584,9 @@ es: edit: 'Editando' usuaries: index: 'Usuaries' + stats: + index: 'Estadísticas' + blazer: + queries: + show: + empty: '(vacío)' From 49e5603687af77ac18d20a1f2975baed18e98925 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 17:57:40 -0300 Subject: [PATCH 004/164] Por cuidados, modificar Blazer pero no poder crear consultas --- app/controllers/concerns/blazer_decorator.rb | 44 ++++++++++++++++++++ config/application.rb | 7 ++++ 2 files changed, 51 insertions(+) create mode 100644 app/controllers/concerns/blazer_decorator.rb diff --git a/app/controllers/concerns/blazer_decorator.rb b/app/controllers/concerns/blazer_decorator.rb new file mode 100644 index 00000000..99b162e6 --- /dev/null +++ b/app/controllers/concerns/blazer_decorator.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# Modificaciones para Blazer +module BlazerDecorator + # No poder obtener información de la base de datos. + module DisableDatabaseInfo + extend ActiveSupport::Concern + + included do + def docs; end + + def tables; end + + def schema; end + end + end + + # Deshabilitar edición de consultas y chequeos. + module DisableEdits + extend ActiveSupport::Concern + + included do + def create; end + + def update; end + + def destroy; end + + def run; end + + def refresh; end + + def cancel; end + end + end +end + +classes = [Blazer::QueriesController, Blazer::ChecksController, Blazer::DashboardsController] +modules = [BlazerDecorator::DisableDatabaseInfo, BlazerDecorator::DisableEdits] +classes.each do |klass| + modules.each do |modul| + klass.include modul unless klass.included_modules.include? modul + end +end diff --git a/config/application.rb b/config/application.rb index 7326ae0f..5b6e373c 100644 --- a/config/application.rb +++ b/config/application.rb @@ -38,6 +38,13 @@ module Sutty config.active_storage.variant_processor = :vips + config.to_prepare do + # Load application's model / class decorators + Dir.glob(File.join(File.dirname(__FILE__), '../app/**/*_decorator.rb')) do |c| + Rails.configuration.cache_classes ? require(c) : load(c) + end + end + config.after_initialize do ActiveStorage::DirectUploadsController.include ActiveStorage::AuthenticatedDirectUploadsController From 85ad518d8d4e92fe0254b43e8775b7c0f2b76830 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 18:01:34 -0300 Subject: [PATCH 005/164] =?UTF-8?q?Correr=20Blazer=20sincr=C3=B3nicamente?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deshabilitamos la funcionalidad asincrónica de Blazer porque ejecuta unos malabares extraños con JS, que filtran la consulta hacia el código. La idea es ejecutar consultas más livianas, con lo que por ahora no lo necesitamos. Podríamos recuperar esto usando ActionCable luego. Además, reimplementa la lógica de generación de gráficos en el controlador, para simplificar la vista. --- app/controllers/concerns/blazer_decorator.rb | 145 +++++++++++++++++++ app/views/blazer/queries/show.haml | 51 +++++++ package.json | 2 + yarn.lock | 35 ++++- 4 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 app/views/blazer/queries/show.haml diff --git a/app/controllers/concerns/blazer_decorator.rb b/app/controllers/concerns/blazer_decorator.rb index 99b162e6..f2598573 100644 --- a/app/controllers/concerns/blazer_decorator.rb +++ b/app/controllers/concerns/blazer_decorator.rb @@ -33,6 +33,149 @@ module BlazerDecorator def cancel; end end end + + # Blazer hace un gran esfuerzo para ejecutar consultas de forma + # asincrónica pero termina enviándolas por JS. + module RunSync + extend ActiveSupport::Concern + + included do + alias_method :original_show, :show + + include Blazer::BaseHelper + + def show + original_show + + options = { user: blazer_user, query: @query, run_id: SecureRandom.uuid, async: false } + @data_source = Blazer.data_sources[@query.data_source] + @result = Blazer::RunStatement.new.perform(@data_source, @statement, options) + chart_data + end + + private + + # blazer-2.4.2/app/views/blazer/queries/run.html.erb + def chart_type + case @result.chart_type + when /\Aline(2)?\z/ + chart_options.merge! min: nil + when /\Abar(2)?\z/ + chart_options.merge! library: { tooltips: { intersect: false, axis: 'x' } } + when 'pie' + chart_options + when 'scatter' + chart_options.merge! library: { tooltips: { intersect: false } }, xtitle: @result.columns[0], + ytitle: @result.columns[1] + when nil + else + if @result.column_types.size == 2 + chart_options.merge! library: { tooltips: { intersect: false, axis: 'x' } } + else + chart_options.merge! library: { tooltips: { intersect: false } } + end + end + + @result.chart_type + end + + def chart_data + @chart_data ||= + case chart_type + when 'line' + @result.columns[1..-1].each_with_index.map do |k, i| + { + name: blazer_series_name(k), + data: @result.rows.map do |r| + [r[0], r[i + 1]] + end, + library: series_library[i] + } + end + when 'line2' + @result.rows.group_by do |r| + v = r[1] + (@result.boom[@result.columns[1]] || {})[v.to_s] || v + end.each_with_index.map do |(name, v), i| + { + name: blazer_series_name(name), + data: v.map do |v2| + [v2[0], v2[2]] + end, + library: series_library[i] + } + end + when 'pie' + @result.rows.map do |r| + [(@result.boom[@result.columns[0]] || {})[r[0].to_s] || r[0], r[1]] + end + when 'bar' + (@result.rows.first.size - 1).times.map do |i| + name = @result.columns[i + 1] + + { + name: blazer_series_name(name), + data: @result.rows.first(20).map do |r| + [(@result.boom[@result.columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]] + end + } + end + when 'bar2' + first_20 = @result.rows.group_by { |r| r[0] }.values.first(20).flatten(1) + labels = first_20.map { |r| r[0] }.uniq + series = first_20.map { |r| r[1] }.uniq + labels.each do |l| + series.each do |s| + first_20 << [l, s, 0] unless first_20.find { |r| r[0] == l && r[1] == s } + end + end + + first_20.group_by do |r| + v = r[1] + (@result.boom[@result.columns[1]] || {})[v.to_s] || v + end.each_with_index.map do |(name, v), _i| + { + name: blazer_series_name(name), + data: v.sort_by do |r2| + labels.index(r2[0]) + end.map do |v2| + v3 = v2[0] + [(@result.boom[@result.columns[0]] || {})[v3.to_s] || v3, v2[2]] + end + } + end + when 'scatter' + @result.rows + end + end + + def target_index + @target_index ||= @result.columns.index do |k| + k.downcase == 'target' + end + end + + def series_library + @series_library ||= {}.tap do |sl| + if target_index + color = '#109618' + sl[target_index - 1] = { + pointStyle: 'line', + hitRadius: 5, + borderColor: color, + pointBackgroundColor: color, + backgroundColor: color, + pointHoverBackgroundColor: color + } + end + end + end + + def chart_options + @chart_options ||= { id: SecureRandom.hex } + end + end + end end classes = [Blazer::QueriesController, Blazer::ChecksController, Blazer::DashboardsController] @@ -42,3 +185,5 @@ classes.each do |klass| klass.include modul unless klass.included_modules.include? modul end end + +Blazer::QueriesController.include BlazerDecorator::RunSync diff --git a/app/views/blazer/queries/show.haml b/app/views/blazer/queries/show.haml new file mode 100644 index 00000000..5ec3da98 --- /dev/null +++ b/app/views/blazer/queries/show.haml @@ -0,0 +1,51 @@ +- blazer_title @query.name +.container + .row + .col-12 + %h1= @query.name + - if @query.description.present? + %p.lead= @query.description + - unless @result.chart_type.blank? + .col-12 + - case @result.chart_type + - when 'line' + = line_chart @chart_data, **@chart_options + - when 'line2' + = line_chart @chart_data, **@chart_options + - when 'pie' + = pie_chart @chart_data, **@chart_options + - when 'bar' + = column_chart @chart_data, **@chart_options + - when 'bar2' + = column_chart @chart_data, **@chart_options + - when 'scatter' + = scatter_chart @chart_data, **@chart_options + .col-12 + %table.table + %thead + %tr + - @result.columns.each do |key| + - next if key.include? 'ciphertext' + - next if key.include? 'encrypted' + %th.position-sticky.background-white= key + %tbody + - @result.rows.each do |row| + %tr + - row.each_with_index do |v, i| + - k = @result.columns[i] + - next if k.include? 'ciphertext' + - next if k.include? 'encrypted' + %td + - if v.is_a?(Time) + - v = blazer_time_value(@data_source, k, v) + + - unless v.nil? + - if v.is_a?(String) && v.empty? + %span.text-muted= t('.empty') + - elsif @data_source.linked_columns[k] + = link_to blazer_format_value(k, v), @data_source.linked_columns[k].gsub('{value}', u(v.to_s)), target: '_blank' + - else + = blazer_format_value(k, v) + + - if (v2 = (@result.boom[k] || {})[v.nil? ? v : v.to_s]) + %span.text-muted= v2 diff --git a/package.json b/package.json index 0a2458a6..6340651f 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,8 @@ "@rails/ujs": "^6.1.3-1", "@rails/webpacker": "5.2.1", "babel-loader": "^8.2.2", + "chart.js": "2.9.3", + "chartkick": "3.2.1", "circular-dependency-plugin": "^5.2.2", "commonmark": "^0.29.0", "fork-awesome": "^1.1.7", diff --git a/yarn.lock b/yarn.lock index 11ff78cb..b478ff67 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2119,6 +2119,34 @@ chalk@^4.1.0: ansi-styles "^4.1.0" supports-color "^7.1.0" +chart.js@2.9.3: + version "2.9.3" + resolved "https://registry.yarnpkg.com/chart.js/-/chart.js-2.9.3.tgz#ae3884114dafd381bc600f5b35a189138aac1ef7" + integrity sha512-+2jlOobSk52c1VU6fzkh3UwqHMdSlgH1xFv9FKMqHiNCpXsGPQa/+81AFa+i3jZ253Mq9aAycPwDjnn1XbRNNw== + dependencies: + chartjs-color "^2.1.0" + moment "^2.10.2" + +chartjs-color-string@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/chartjs-color-string/-/chartjs-color-string-0.6.0.tgz#1df096621c0e70720a64f4135ea171d051402f71" + integrity sha512-TIB5OKn1hPJvO7JcteW4WY/63v6KwEdt6udfnDE9iCAZgy+V4SrbSxoIbTw/xkUIapjEI4ExGtD0+6D3KyFd7A== + dependencies: + color-name "^1.0.0" + +chartjs-color@^2.1.0: + version "2.4.1" + resolved "https://registry.yarnpkg.com/chartjs-color/-/chartjs-color-2.4.1.tgz#6118bba202fe1ea79dd7f7c0f9da93467296c3b0" + integrity sha512-haqOg1+Yebys/Ts/9bLo/BqUcONQOdr/hoEr2LLTRl6C5LXctUdHxsCYfvQVg5JIxITrfCNUDr4ntqmQk9+/0w== + dependencies: + chartjs-color-string "^0.6.0" + color-convert "^1.9.3" + +chartkick@3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/chartkick/-/chartkick-3.2.1.tgz#a80c2005ae353c5ae011d0a756b6f592fc8fc7a9" + integrity sha512-zV0kUeZNqrX28AmPt10QEDXHKadbVFOTAFkCMyJifHzGFkKzGCDXxVR8orZ0fC1HbePzRn5w6kLCOVxDQbMUCg== + chokidar@^2.1.8: version "2.1.8" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.8.tgz#804b3a7b6a99358c3c5c61e71d8728f041cff917" @@ -2238,7 +2266,7 @@ collection-visit@^1.0.0: map-visit "^1.0.0" object-visit "^1.0.0" -color-convert@^1.9.0, color-convert@^1.9.1: +color-convert@^1.9.0, color-convert@^1.9.1, color-convert@^1.9.3: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== @@ -5005,6 +5033,11 @@ mkdirp@^1.0.3, mkdirp@^1.0.4: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== +moment@^2.10.2: + version "2.29.1" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3" + integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ== + move-concurrently@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92" From 75e6b7a8014ea83362e126dc85dcd681c053514d Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 18:25:42 -0300 Subject: [PATCH 006/164] =?UTF-8?q?Agregar=20los=20helpers=20de=20Devise?= =?UTF-8?q?=20tambi=C3=A9n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Para que tengamos `current_usuarie` dentro de las vistas de Blazer. --- app/controllers/application_controller.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 2b5088f7..efe1a4a3 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -71,7 +71,9 @@ class ApplicationController < ActionController::Base end # Necesario para los breadcrumbs. - ActionView::Base.include Loaf::ViewExtensions unless ActionView::Base.included_modules.include? Loaf::ViewExtensions + [Loaf::ViewExtensions, Devise::Controllers::UrlHelpers].each do |helper_module| + ActionView::Base.include helper_module unless ActionView::Base.included_modules.include? helper_module + end breadcrumb current_usuarie.email, main_app.edit_usuarie_registration_path breadcrumb 'sites.index', main_app.sites_path, match: :exact From 841279f6cf81d545f064e94dc3aab39e7d2cc454 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 18:31:17 -0300 Subject: [PATCH 007/164] =?UTF-8?q?fixup!=20Agregar=20los=20helpers=20de?= =?UTF-8?q?=20Devise=20tambi=C3=A9n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/application_controller.rb | 4 +--- app/views/layouts/_breadcrumb.haml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index efe1a4a3..2b5088f7 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -71,9 +71,7 @@ class ApplicationController < ActionController::Base end # Necesario para los breadcrumbs. - [Loaf::ViewExtensions, Devise::Controllers::UrlHelpers].each do |helper_module| - ActionView::Base.include helper_module unless ActionView::Base.included_modules.include? helper_module - end + ActionView::Base.include Loaf::ViewExtensions unless ActionView::Base.included_modules.include? Loaf::ViewExtensions breadcrumb current_usuarie.email, main_app.edit_usuarie_registration_path breadcrumb 'sites.index', main_app.sites_path, match: :exact diff --git a/app/views/layouts/_breadcrumb.haml b/app/views/layouts/_breadcrumb.haml index f209787c..dc0e3158 100644 --- a/app/views/layouts/_breadcrumb.haml +++ b/app/views/layouts/_breadcrumb.haml @@ -12,7 +12,7 @@ - else %span.line-clamp-1= link_to crumb.name, crumb.url - - if current_usuarie + - if @current_usuarie || current_usuarie %ul.navbar-nav - if @site&.tienda? %li.nav-item From af67c39dc477581fb6a0acd2adce5aacb683d0aa Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 18:37:15 -0300 Subject: [PATCH 008/164] Agregar el nombre del sitio en la miga de pan --- app/controllers/application_controller.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 2b5088f7..c9e5a999 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -65,7 +65,8 @@ class ApplicationController < ActionController::Base # Necesario para poder acceder a Blazer. Solo les usuaries de este # sitio pueden acceder al panel. def require_usuarie - unless find_site.usuarie? current_usuarie + site = find_site + unless site.usuarie? current_usuarie redirect_to root_path return end @@ -75,6 +76,7 @@ class ApplicationController < ActionController::Base breadcrumb current_usuarie.email, main_app.edit_usuarie_registration_path breadcrumb 'sites.index', main_app.sites_path, match: :exact + breadcrumb site.title, main_app.site_path(site), match: :exact breadcrumb 'stats.index', root_path, match: :exact end From 44450da5208b55582c2ff287c538d71a452da274 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 18:41:09 -0300 Subject: [PATCH 009/164] Les usuaries solo pueden ver sus propias consultas. --- app/controllers/concerns/blazer_decorator.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/controllers/concerns/blazer_decorator.rb b/app/controllers/concerns/blazer_decorator.rb index f2598573..eee0db64 100644 --- a/app/controllers/concerns/blazer_decorator.rb +++ b/app/controllers/concerns/blazer_decorator.rb @@ -55,6 +55,11 @@ module BlazerDecorator private + # Solo mostrar las consultas de le usuarie + def set_queries(_) + @queries = (@current_usuarie || current_usuarie).blazer_queries + end + # blazer-2.4.2/app/views/blazer/queries/run.html.erb def chart_type case @result.chart_type From 6f08ca6c36b1297d93bd7ba435e5e7d70b930a60 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 3 Aug 2021 10:10:30 -0300 Subject: [PATCH 010/164] =?UTF-8?q?A=20veces=20se=20pasa=20el=20par=C3=A1m?= =?UTF-8?q?etro=20en=20set=5Fqueries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/concerns/blazer_decorator.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/concerns/blazer_decorator.rb b/app/controllers/concerns/blazer_decorator.rb index eee0db64..876f423d 100644 --- a/app/controllers/concerns/blazer_decorator.rb +++ b/app/controllers/concerns/blazer_decorator.rb @@ -56,7 +56,7 @@ module BlazerDecorator private # Solo mostrar las consultas de le usuarie - def set_queries(_) + def set_queries(_ = nil) @queries = (@current_usuarie || current_usuarie).blazer_queries end From 489cbb414ccd4a7fbd5c81f7e6051838a4512ada Mon Sep 17 00:00:00 2001 From: f Date: Tue, 3 Aug 2021 10:15:48 -0300 Subject: [PATCH 011/164] Ya no usamos un hash de queries --- app/views/blazer/queries/home.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/blazer/queries/home.haml b/app/views/blazer/queries/home.haml index 325ba121..977b6bda 100644 --- a/app/views/blazer/queries/home.haml +++ b/app/views/blazer/queries/home.haml @@ -6,4 +6,4 @@ -# Por alguna razón no tenemos acceso a query_path para poder generar la URL según Rails - %td= link_to query[:name], "/sites/#{params[:site_id]}/stats/queries/#{query[:to_param]}" + %td= link_to query[:name], "/sites/#{params[:site_id]}/stats/queries/#{query.to_param}" From e87fad33eae8b1225d090a9e6f5464933c69dea6 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 3 Aug 2021 10:16:26 -0300 Subject: [PATCH 012/164] =?UTF-8?q?La=20tabla=20se=20puede=20scrollear=20y?= =?UTF-8?q?=20mantiene=20los=20t=C3=ADtulos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/blazer/queries/show.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/blazer/queries/show.haml b/app/views/blazer/queries/show.haml index 5ec3da98..3b2beed6 100644 --- a/app/views/blazer/queries/show.haml +++ b/app/views/blazer/queries/show.haml @@ -27,7 +27,7 @@ - @result.columns.each do |key| - next if key.include? 'ciphertext' - next if key.include? 'encrypted' - %th.position-sticky.background-white= key + %th.position-sticky.background-white{ style: 'top: 0' }= key %tbody - @result.rows.each do |row| %tr From 71ff9e5e7b22013b5c4f033518871ea9670d52c5 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 3 Aug 2021 10:24:18 -0300 Subject: [PATCH 013/164] =?UTF-8?q?Traducir=20columnas=20o=20mostrarlas=20?= =?UTF-8?q?como=20t=C3=ADtulos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Los nombres más comunas de columnas se pueden agregar al archivo de traducción, si la traducción no existe se convierte a un título. ```ruby "total_horas".titleize => "Total Horas" ``` --- app/views/blazer/queries/show.haml | 2 +- config/locales/en.yml | 5 +++++ config/locales/es.yml | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/app/views/blazer/queries/show.haml b/app/views/blazer/queries/show.haml index 3b2beed6..3b5cb152 100644 --- a/app/views/blazer/queries/show.haml +++ b/app/views/blazer/queries/show.haml @@ -27,7 +27,7 @@ - @result.columns.each do |key| - next if key.include? 'ciphertext' - next if key.include? 'encrypted' - %th.position-sticky.background-white{ style: 'top: 0' }= key + %th.position-sticky.background-white{ style: 'top: 0' }= t("blazer.columns.#{key}", default: key.titleize) %tbody - @result.rows.each do |row| %tr diff --git a/config/locales/en.yml b/config/locales/en.yml index 7fd83f63..18faa8bb 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -579,6 +579,11 @@ en: stats: index: 'Statistics' blazer: + columns: + total: 'Total' + dia: 'Date' + date: 'Date' + visitas: 'Visits' queries: show: empty: '(empty)' diff --git a/config/locales/es.yml b/config/locales/es.yml index 0a48742a..5229a591 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -587,6 +587,11 @@ es: stats: index: 'Estadísticas' blazer: + columns: + total: 'Total' + dia: 'Fecha' + date: 'Fecha' + visitas: 'Visitas' queries: show: empty: '(vacío)' From 71436d3be49cb3d0cdde8e37338369f10fdaf81f Mon Sep 17 00:00:00 2001 From: f Date: Wed, 4 Aug 2021 12:17:49 -0300 Subject: [PATCH 014/164] =?UTF-8?q?Usar=20el=20sistema=20de=20autorizaci?= =?UTF-8?q?=C3=B3n=20de=20Sutty?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Y eliminar código sin utilizar. --- app/controllers/application_controller.rb | 9 ++++----- app/controllers/posts_controller.rb | 3 --- app/controllers/private_controller.rb | 2 -- app/controllers/sites_controller.rb | 3 --- app/controllers/stats_controller.rb | 18 ------------------ app/models/{site_stat.rb => site_blazer.rb} | 2 +- app/policies/site_blazer_policy.rb | 10 ++++++++++ app/policies/site_stat_policy.rb | 15 --------------- 8 files changed, 15 insertions(+), 47 deletions(-) delete mode 100644 app/controllers/stats_controller.rb rename app/models/{site_stat.rb => site_blazer.rb} (50%) create mode 100644 app/policies/site_blazer_policy.rb delete mode 100644 app/policies/site_stat_policy.rb diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index c9e5a999..d8498218 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -3,6 +3,7 @@ # Forma de ingreso a Sutty class ApplicationController < ActionController::Base include ExceptionHandler + include Pundit protect_from_forgery with: :null_session, prepend: true @@ -10,6 +11,7 @@ class ApplicationController < ActionController::Base before_action :configure_permitted_parameters, if: :devise_controller? around_action :set_locale + rescue_from Pundit::NilPolicyError, with: :page_not_found rescue_from ActionController::RoutingError, with: :page_not_found rescue_from ActionController::ParameterMissing, with: :page_not_found @@ -33,7 +35,7 @@ class ApplicationController < ActionController::Base def find_site id = params[:site_id] || params[:id] - unless (site = current_usuarie.sites.find_by_name(id)) + unless (site = current_usuarie&.sites&.find_by_name(id)) raise SiteNotFound end @@ -66,10 +68,7 @@ class ApplicationController < ActionController::Base # sitio pueden acceder al panel. def require_usuarie site = find_site - unless site.usuarie? current_usuarie - redirect_to root_path - return - end + authorize SiteBlazer.new(site) # Necesario para los breadcrumbs. ActionView::Base.include Loaf::ViewExtensions unless ActionView::Base.included_modules.include? Loaf::ViewExtensions diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index 3ef26720..448592de 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -2,9 +2,6 @@ # Controlador para artículos class PostsController < ApplicationController - include Pundit - rescue_from Pundit::NilPolicyError, with: :page_not_found - before_action :authenticate_usuarie! # TODO: Traer los comunes desde ApplicationController diff --git a/app/controllers/private_controller.rb b/app/controllers/private_controller.rb index bb4d782d..01b6888c 100644 --- a/app/controllers/private_controller.rb +++ b/app/controllers/private_controller.rb @@ -6,8 +6,6 @@ class PrivateController < ApplicationController # XXX: Permite ejecutar JS skip_forgery_protection - include Pundit - # Enviar el archivo si existe, agregar una / al final siempre para no # romper las direcciones relativas. def show diff --git a/app/controllers/sites_controller.rb b/app/controllers/sites_controller.rb index bdaa9011..b4826226 100644 --- a/app/controllers/sites_controller.rb +++ b/app/controllers/sites_controller.rb @@ -2,9 +2,6 @@ # Controlador de sitios class SitesController < ApplicationController - include Pundit - rescue_from Pundit::NilPolicyError, with: :page_not_found - before_action :authenticate_usuarie! breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb deleted file mode 100644 index 07baaf1a..00000000 --- a/app/controllers/stats_controller.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -# Estadísticas del sitio -class StatsController < ApplicationController - include Pundit - before_action :authenticate_usuarie! - - def index - @site = find_site - authorize SiteStat.new(@site) - - # Solo queremos el promedio de tiempo de compilación, no de - # instalación de dependencias. - stats = @site.build_stats.jekyll - @build_avg = stats.average(:seconds).to_f.round(2) - @build_max = stats.maximum(:seconds).to_f.round(2) - end -end diff --git a/app/models/site_stat.rb b/app/models/site_blazer.rb similarity index 50% rename from app/models/site_stat.rb rename to app/models/site_blazer.rb index 73503aca..76dee12a 100644 --- a/app/models/site_stat.rb +++ b/app/models/site_blazer.rb @@ -1,3 +1,3 @@ # frozen_string_literal: true -SiteStat = Struct.new(:site) +SiteBlazer = Struct.new(:site) diff --git a/app/policies/site_blazer_policy.rb b/app/policies/site_blazer_policy.rb new file mode 100644 index 00000000..a6ea01b7 --- /dev/null +++ b/app/policies/site_blazer_policy.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +# Les invitades no pueden ver las estadísticas (aun) +SiteBlazerPolicy = Struct.new(:usuarie, :site_blazer) do + def home? + site_blazer&.site&.usuarie? usuarie + end + + alias_method :show?, :home? +end diff --git a/app/policies/site_stat_policy.rb b/app/policies/site_stat_policy.rb deleted file mode 100644 index a797034c..00000000 --- a/app/policies/site_stat_policy.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -# Política de acceso a las estadísticas -class SiteStatPolicy - attr_reader :site_stat, :usuarie - - def initialize(usuarie, site_stat) - @usuarie = usuarie - @site_stat = site_stat - end - - def index? - site_stat.site.usuarie? usuarie - end -end From 249b115af848f02fb11dd1bbebd55c3a31f5fa28 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 4 Aug 2021 20:16:56 -0300 Subject: [PATCH 015/164] No enviar URLs en los chequeos. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Necesita más trabajo hacer esto y no tenemos un uso inmediato para chequeos específicos de sitios. fixes #2331 --- .../blazer/check_mailer/failing_checks.haml | 5 ++++ .../blazer/check_mailer/state_change.haml | 30 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 app/views/blazer/check_mailer/failing_checks.haml create mode 100644 app/views/blazer/check_mailer/state_change.haml diff --git a/app/views/blazer/check_mailer/failing_checks.haml b/app/views/blazer/check_mailer/failing_checks.haml new file mode 100644 index 00000000..8624ae9b --- /dev/null +++ b/app/views/blazer/check_mailer/failing_checks.haml @@ -0,0 +1,5 @@ +%ul + - @checks.each do |check| + %li + = link_to check.query.name + = check.state diff --git a/app/views/blazer/check_mailer/state_change.haml b/app/views/blazer/check_mailer/state_change.haml new file mode 100644 index 00000000..48418a58 --- /dev/null +++ b/app/views/blazer/check_mailer/state_change.haml @@ -0,0 +1,30 @@ +!!! +%html + %head + %meta{:content => "text/html; charset=UTF-8", "http-equiv" => "Content-Type"}/ + %body{:style => "font-family: 'Helvetica Neue', Arial, Helvetica; font-size: 14px; color: #333;"} + - if @error + %p= @error + - elsif @rows_count > 0 && @check_type == "bad_data" + %p + - if @rows_count <= 10 + = pluralize(@rows_count, "row") + - else + Showing 10 of #{@rows_count} rows + %table{:style => "width: 100%; border-spacing: 0; border-collapse: collapse;"} + %thead + %tr + - @columns.first(5).each do |column| + %th{:style => "padding: 8px; line-height: 1.4; text-align: left; vertical-align: bottom; border-bottom: 2px solid #ddd; width: #{(100 / @columns.size).round(2)}%;"} + = column + %tbody + - @rows.first(10).each do |row| + %tr + - @columns.first(5).each_with_index do |column, i| + %td{:style => "padding: 8px; line-height: 1.4; vertical-align: top; border-top: 1px solid #ddd;"} + - value = row[i] + - if @column_types[i] == "time" && value.to_s.length > 10 + - value = Time.parse(value).in_time_zone(Blazer.time_zone) rescue value + = value + - if @columns.size > 5 + %p{:style => "color: #999;"} Only first 5 columns shown From 5885dd7e9615288bbe36cf0b413c3aeda042853e Mon Sep 17 00:00:00 2001 From: f Date: Fri, 6 Aug 2021 17:47:27 -0300 Subject: [PATCH 016/164] fixup! No enviar URLs en los chequeos. --- app/views/blazer/check_mailer/failing_checks.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/blazer/check_mailer/failing_checks.haml b/app/views/blazer/check_mailer/failing_checks.haml index 8624ae9b..c28c3936 100644 --- a/app/views/blazer/check_mailer/failing_checks.haml +++ b/app/views/blazer/check_mailer/failing_checks.haml @@ -1,5 +1,5 @@ %ul - @checks.each do |check| %li - = link_to check.query.name + = check.query.name = check.state From 460635e940fca637bf41499cb8d4070beb9a3db7 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:03:02 -0300 Subject: [PATCH 017/164] ya no hace falta analizar archivos en primer plano --- config/initializers/analyze_job.rb | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 config/initializers/analyze_job.rb diff --git a/config/initializers/analyze_job.rb b/config/initializers/analyze_job.rb deleted file mode 100644 index f268e0dd..00000000 --- a/config/initializers/analyze_job.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -# TODO: Estamos procesando el análisis de los archivos en el momento -# porque queremos obtener la ruta del archivo en el momento y no -# después. Necesitaríamos poder generar el vínculo en el -# repositorio a destiempo, modificando el Job de ActiveStorage -ActiveStorage::AnalyzeJob.queue_adapter = :inline From 29950e4380b4c6d5122a0bfd3f8024178b44a40d Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:05:11 -0300 Subject: [PATCH 018/164] cargar decorators --- config/application.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/application.rb b/config/application.rb index 7326ae0f..0e6e7f44 100644 --- a/config/application.rb +++ b/config/application.rb @@ -38,6 +38,12 @@ module Sutty config.active_storage.variant_processor = :vips + config.to_prepare do + Dir.glob(File.join(File.dirname(__FILE__), '..', 'app', '**', '*_decorator.rb')) do |c| + Rails.configuration.cache_classes ? require(c) : load(c) + end + end + config.after_initialize do ActiveStorage::DirectUploadsController.include ActiveStorage::AuthenticatedDirectUploadsController From 3de1228c26541185a62fa7a528dfc170308018bf Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:08:09 -0300 Subject: [PATCH 019/164] implementa un servicio de carga de archivos en jekyll --- .../active_storage/service/jekyll_service.rb | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 app/lib/active_storage/service/jekyll_service.rb diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb new file mode 100644 index 00000000..686c3979 --- /dev/null +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module ActiveStorage + module Service + # Sube los archivos a cada repositorio y los agrega al LFS de su + # repositorio git. + # + # @todo: Implementar LFS. No nos gusta mucho la idea porque duplica + # el espacio en disco, pero es la única forma que tenemos (hasta que + # implementemos IPFS) para poder transferir los archivos junto con el + # sitio. + class JekyllService < Service::DiskService + BLOB_NAME = 'blob' + + # Para poder guardar el archivo con el nombre original pero poder + # recuperarlo durante el download, luego de subirlo le cambiamos el + # nombre y creamos un link simbólico a un nombre conocido. + def upload(key, io, checksum: nil, **options) + super.tap do + path = path_for(key) + filename = options[:filename].to_s + + FileUtils.mv path, path.sub(/#{BLOB_NAME}\z/, filename) + FileUtils.ln_s filename, path + end + end + + # Mantener retrocompatibilidad con cómo gestionamos los archivos + # subidos hasta ahora. + # + # @param :key [String] + # @return [String] + def folder_for(key) + key + end + + # Crea una ruta para la llave con un nombre conocido. + def path_for(key) + File.join root, folder_for(key), BLOB_NAME + end + end + end +end From 873f2c7bcb8db83a1bdc46580158ff28ce3e67e9 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:09:06 -0300 Subject: [PATCH 020/164] informar cuando no se pudo cargar el archivo --- app/models/metadata_file.rb | 1 + config/locales/en.yml | 2 ++ config/locales/es.yml | 2 ++ 3 files changed, 5 insertions(+) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 80cefa27..b020d078 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -19,6 +19,7 @@ class MetadataFile < MetadataTemplate errors << I18n.t("metadata.#{type}.site_invalid") if site.invalid? errors << I18n.t("metadata.#{type}.path_required") if path_missing? errors << I18n.t("metadata.#{type}.no_file_for_description") if no_file_for_description? + errors << I18n.t("metadata.#{type}.attachment_missing") unless static_file errors.compact! errors.empty? diff --git a/config/locales/en.yml b/config/locales/en.yml index b814796d..8bbe3621 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -41,10 +41,12 @@ en: not_an_image: 'Not an image' path_required: 'Missing image for upload' no_file_for_description: "Description with no associated image" + attachment_missing: "I couldn't save the image :(" file: site_invalid: 'The file cannot be stored if the site configuration is not valid' path_required: "Missing file for upload" no_file_for_description: "Description with no associated file" + attachment_missing: "I couldn't save the file :(" event: zone_missing: 'Inexistent timezone' date_missing: 'Event date is required' diff --git a/config/locales/es.yml b/config/locales/es.yml index a6fbd407..93dd162e 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -41,10 +41,12 @@ es: not_an_image: 'No es una imagen' path_required: 'Se necesita una imagen' no_file_for_description: 'Se envió una descripción sin imagen asociada' + attachment_missing: 'no pude guardar el archivo :(' file: site_invalid: 'El archivo no se puede almacenar si la configuración del sitio no es válida' path_required: 'Se necesita un archivo' no_file_for_description: 'se envió una descripción sin archivo asociado' + attachment_missing: 'no pude guardar el archivo :(' event: zone_missing: 'El huso horario no es correcto' date_missing: 'La fecha es obligatoria' From c4139c4b92746c84cdbdaaffe1fea31a69afc9a8 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:12:06 -0300 Subject: [PATCH 021/164] =?UTF-8?q?eliminar=20c=C3=B3digo=20que=20no=20se?= =?UTF-8?q?=20usa?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/metadata_file.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index b020d078..58adc857 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -35,12 +35,6 @@ class MetadataFile < MetadataTemplate value['path'].is_a?(String) end - # Determina si la ruta es opcional pero deja pasar si la ruta se - # especifica - def path_optional? - !required && !path? - end - # Asociar la imagen subida al sitio y obtener la ruta # # XXX: Si evitamos guardar cambios con changed? no tenemos forma de From 3d5267451eec0e879fad28ef4086cd240884d683 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:15:38 -0300 Subject: [PATCH 022/164] =?UTF-8?q?eliminar=20c=C3=B3digo=20que=20ya=20no?= =?UTF-8?q?=20se=20usa?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/metadata_file.rb | 65 ------------------------------------- 1 file changed, 65 deletions(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 58adc857..86644fcc 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -93,71 +93,6 @@ class MetadataFile < MetadataTemplate private - def filemagic - @filemagic ||= FileMagic.new(FileMagic::MAGIC_MIME) - end - - # @return [Pathname] - def path - @path ||= Pathname.new(File.join(site.path, value['path'])) - end - - def file - return unless path? - - @file ||= - case value['path'] - when ActionDispatch::Http::UploadedFile then value['path'].tempfile.path - when String then File.join(site.path, value['path']) - end - end - - # Hacemos un link duro para colocar el archivo dentro del repositorio - # y no duplicar el espacio que ocupan. Esto requiere que ambos - # directorios estén dentro del mismo punto de montaje. - # - # XXX: Asumimos que el archivo destino no existe porque siempre - # contiene una key única. - # - # @return [Boolean] - def hardlink - return if hardlink? - return if File.exist? destination_path - - FileUtils.mkdir_p(File.dirname(destination_path)) - FileUtils.ln(uploaded_path, destination_path).zero? - end - - def hardlink? - File.stat(uploaded_path).ino == File.stat(destination_path).ino - rescue Errno::ENOENT - false - end - - # Obtener la ruta al archivo - # https://stackoverflow.com/a/53908358 - def uploaded_relative_path - ActiveStorage::Blob.service.path_for(static_file.key) - end - - # @return [String] - def uploaded_path - Rails.root.join uploaded_relative_path - end - - # La ruta del archivo mantiene el nombre original pero contiene el - # nombre interno y único del archivo para poder relacionarlo con el - # archivo subido en Sutty. - # - # @return [String] - def relative_destination_path - @relative_destination_path ||= File.join('public', static_file.key, static_file.filename.to_s) - end - - # @return [String] - def destination_path - @destination_path ||= File.join(site.path, relative_destination_path) - end # No hay archivo pero se lo describió def no_file_for_description? From d8fd25a7cb32ab6cdec028f4d054cf398df4b97c Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:16:36 -0300 Subject: [PATCH 023/164] =?UTF-8?q?chequear=20la=20descripci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/metadata_file.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 86644fcc..20d093b2 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -91,11 +91,14 @@ class MetadataFile < MetadataTemplate value['path'].present? end - private + def description? + value['description'].present? + end + private # No hay archivo pero se lo describió def no_file_for_description? - value['description'].present? && value['path'].blank? + !path? && description? end end From 63f8b869eb185dd9c607ed4b684baf9ab478a08d Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:17:03 -0300 Subject: [PATCH 024/164] fixup! cargar decorators --- config/application.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/application.rb b/config/application.rb index 0e6e7f44..031c1909 100644 --- a/config/application.rb +++ b/config/application.rb @@ -39,7 +39,7 @@ module Sutty config.active_storage.variant_processor = :vips config.to_prepare do - Dir.glob(File.join(File.dirname(__FILE__), '..', 'app', '**', '*_decorator.rb')) do |c| + Dir.glob(File.join(File.dirname(__FILE__), '..', 'app', '**', '*_decorator.rb')).sort.each do |c| Rails.configuration.cache_classes ? require(c) : load(c) end end From 37deb361785d9f79c2d721898c3545a5c7323985 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:19:40 -0300 Subject: [PATCH 025/164] conseguir la ruta al archivo --- app/models/metadata_file.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 20d093b2..89e6d461 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -83,6 +83,15 @@ class MetadataFile < MetadataTemplate end end + # Obtiene la ruta absoluta al archivo + # + # @return [Pathname] + def pathname + raise NoMethodError unless uploaded? + + @pathname ||= Pathname.new(File.join(site.path, value['path'])) + end + def key_from_path path.dirname.basename.to_s end From 03d9913f58f4be0bd0e2c3417577bab6f8a67c4a Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:21:03 -0300 Subject: [PATCH 026/164] obtener el archivo subido --- app/models/metadata_file.rb | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 89e6d461..e4c0d037 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -66,18 +66,19 @@ class MetadataFile < MetadataTemplate # XXX: La última opción provoca archivos duplicados, pero es lo mejor # que tenemos hasta que resolvamos https://0xacab.org/sutty/sutty/-/issues/213 # - # @return [ActiveStorage::Attachment] + # @todo encontrar una forma de obtener el attachment sin tener que + # recurrir al último subido. + # + # @return [ActiveStorage::Attachment,nil] def static_file - return unless path? - @static_file ||= case value['path'] when ActionDispatch::Http::UploadedFile site.static_files.last if site.static_files.attach(value['path']) when String - if (blob = ActiveStorage::Blob.where(key: key_from_path).pluck(:id).first) - site.static_files.find_by(blob_id: blob) - elsif site.static_files.attach(io: path.open, filename: path.basename) + if (blob_id = ActiveStorage::Blob.where(key: key_from_path).pluck(:id).first) + site.static_files.find_by(blob_id: blob_id) + elsif path? && pathname.exist? && site.static_files.attach(io: pathname.open, filename: pathname.basename) site.static_files.last end end @@ -92,8 +93,11 @@ class MetadataFile < MetadataTemplate @pathname ||= Pathname.new(File.join(site.path, value['path'])) end + # Obtiene la key del attachment a partir de la ruta + # + # @return [String] def key_from_path - path.dirname.basename.to_s + pathname.dirname.basename.to_s end def path? From fdbe724f7c92953e9ba2cfa74f8470a6fa617c2e Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:21:26 -0300 Subject: [PATCH 027/164] asociar el archivo subido al post --- app/models/metadata_file.rb | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index e4c0d037..1c859481 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -43,13 +43,7 @@ class MetadataFile < MetadataTemplate # repetida. def save value['description'] = sanitize value['description'] - - if path? - hardlink - value['path'] = relative_destination_path - else - value['path'] = nil - end + value['path'] = static_file ? relative_destination_path_with_filename.to_s : nil true end @@ -110,6 +104,28 @@ class MetadataFile < MetadataTemplate private + # Obtener la ruta al archivo relativa al sitio + # + # @return [Pathname] + def destination_path + Pathname.new(static_file_path) + end + + # Agrega el nombre de archivo a la ruta para tener retrocompatibilidad + # + # @return [Pathname] + def destination_path_with_filename + destination_path.realpath + end + + def relative_destination_path_with_filename + destination_path_with_filename.relative_path_from(site.path) + end + + def static_file_path + static_file.blob.service.path_for(static_file.key) + end + # No hay archivo pero se lo describió def no_file_for_description? !path? && description? From f3df5504944cbdbf2f07aa0cd33f8fd8cd7f55a0 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:21:48 -0300 Subject: [PATCH 028/164] =?UTF-8?q?no=20permitir=20subir=20im=C3=A1genes?= =?UTF-8?q?=20que=20no=20son=20para=20web?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/metadata_image.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/models/metadata_image.rb b/app/models/metadata_image.rb index f91a6273..f86c5c26 100644 --- a/app/models/metadata_image.rb +++ b/app/models/metadata_image.rb @@ -13,8 +13,6 @@ class MetadataImage < MetadataFile # Determina si es una imagen def image? - return true unless file - - filemagic.file(file).starts_with? 'image/' + static_file&.blob&.send(:web_image?) end end From 02b52b23b91b0c88a9c48b01604eb9ee22692d9f Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:24:04 -0300 Subject: [PATCH 029/164] compartir el nombre de archivo con JekyllService --- app/models/active_storage/blob_decorator.rb | 26 +++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 app/models/active_storage/blob_decorator.rb diff --git a/app/models/active_storage/blob_decorator.rb b/app/models/active_storage/blob_decorator.rb new file mode 100644 index 00000000..9c01251a --- /dev/null +++ b/app/models/active_storage/blob_decorator.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module ActiveStorage + # Modificaciones a ActiveStorage::Blob + module BlobDecorator + extend ActiveSupport::Concern + + included do + # Permitir que llegue el nombre de archivo al servicio de subida de + # archivos. + # + # @return [Hash] + def service_metadata + if forcibly_serve_as_binary? + { content_type: ActiveStorage.binary_content_type, disposition: :attachment, filename: filename } + elsif !allowed_inline? + { content_type: content_type, disposition: :attachment, filename: filename } + else + { content_type: content_type, filename: filename } + end + end + end + end +end + +ActiveStorage::Blob.include ActiveStorage::BlobDecorator From 69e7df4c31b8f3b72d9e2cde2b7430bb3cef4f66 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 4 Mar 2022 19:26:56 -0300 Subject: [PATCH 030/164] instanciar JekyllService para cada sitio --- .../service/registry_decorator.rb | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 app/lib/active_storage/service/registry_decorator.rb diff --git a/app/lib/active_storage/service/registry_decorator.rb b/app/lib/active_storage/service/registry_decorator.rb new file mode 100644 index 00000000..f7f20784 --- /dev/null +++ b/app/lib/active_storage/service/registry_decorator.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module ActiveStorage + module Service + # Modificaciones a ActiveStorage::Service::Registry + module RegistryDecorator + extend ActiveSupport::Concern + + included do + # El mismo comportamiento que #fetch con el agregado de generar + # un {JekyllService} para cada sitio. + def fetch(name) + services.fetch(name.to_sym) do |key| + if configurations.include?(key) + services[key] = configurator.build(key) + elsif (site = Site.find_by_name(key)) + root = File.join(site.path, 'public') + services[key] = ActiveStorage::Service::JekyllService.new(root: root, public: true).tap do |s| + s.name = key.to_sym + end + elsif block_given? + yield key + else + raise KeyError, "Missing configuration for the #{key} Active Storage service. " \ + "Configurations available for the #{configurations.keys.to_sentence} services." + end + end + end + end + end + end +end + +ActiveStorage::Service::Registry.include ActiveStorage::Service::RegistryDecorator From 15e978c877cacc8b5eb3a6c79c5e0e9cc3349eda Mon Sep 17 00:00:00 2001 From: f Date: Sat, 5 Mar 2022 20:08:55 -0300 Subject: [PATCH 031/164] eran clases --- app/lib/active_storage/service/jekyll_service.rb | 2 +- app/lib/active_storage/service/registry_decorator.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb index 686c3979..173f6898 100644 --- a/app/lib/active_storage/service/jekyll_service.rb +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module ActiveStorage - module Service + class Service # Sube los archivos a cada repositorio y los agrega al LFS de su # repositorio git. # diff --git a/app/lib/active_storage/service/registry_decorator.rb b/app/lib/active_storage/service/registry_decorator.rb index f7f20784..f6794607 100644 --- a/app/lib/active_storage/service/registry_decorator.rb +++ b/app/lib/active_storage/service/registry_decorator.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module ActiveStorage - module Service + class Service # Modificaciones a ActiveStorage::Service::Registry module RegistryDecorator extend ActiveSupport::Concern From 7f6063475bb63eed2cab8523d5f15d1ef71b15ea Mon Sep 17 00:00:00 2001 From: f Date: Sat, 5 Mar 2022 20:09:09 -0300 Subject: [PATCH 032/164] =?UTF-8?q?m=C3=A9todo=20gen=C3=A9rico=20para=20in?= =?UTF-8?q?stanciar=20el=20servicio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/lib/active_storage/service/jekyll_service.rb | 7 +++++++ app/lib/active_storage/service/registry_decorator.rb | 5 +---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb index 173f6898..20dda523 100644 --- a/app/lib/active_storage/service/jekyll_service.rb +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -12,6 +12,13 @@ module ActiveStorage class JekyllService < Service::DiskService BLOB_NAME = 'blob' + # Genera un servicio para un sitio determinado + def self.build_for_site(site:) + new(root: File.join(site.path, 'public'), public: true).tap do |js| + js.name = site.name.to_sym + end + end + # Para poder guardar el archivo con el nombre original pero poder # recuperarlo durante el download, luego de subirlo le cambiamos el # nombre y creamos un link simbólico a un nombre conocido. diff --git a/app/lib/active_storage/service/registry_decorator.rb b/app/lib/active_storage/service/registry_decorator.rb index f6794607..c7096356 100644 --- a/app/lib/active_storage/service/registry_decorator.rb +++ b/app/lib/active_storage/service/registry_decorator.rb @@ -14,10 +14,7 @@ module ActiveStorage if configurations.include?(key) services[key] = configurator.build(key) elsif (site = Site.find_by_name(key)) - root = File.join(site.path, 'public') - services[key] = ActiveStorage::Service::JekyllService.new(root: root, public: true).tap do |s| - s.name = key.to_sym - end + services[key] = ActiveStorage::Service::JekyllService.build_for_site(site: site) elsif block_given? yield key else From 10eef47ce8bfb0f233083e070207f7d1c8bbf578 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 5 Mar 2022 20:09:37 -0300 Subject: [PATCH 033/164] todos los archivos subidos se asocian al sitio --- .../attached/changes/create_one_decorator.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 app/lib/active_storage/attached/changes/create_one_decorator.rb diff --git a/app/lib/active_storage/attached/changes/create_one_decorator.rb b/app/lib/active_storage/attached/changes/create_one_decorator.rb new file mode 100644 index 00000000..bfb92478 --- /dev/null +++ b/app/lib/active_storage/attached/changes/create_one_decorator.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module ActiveStorage + module Attached::Changes::CreateOneDecorator + extend ActiveSupport::Concern + + included do + private + + # A partir de ahora todos los archivos se suben al servicio de + # cada sitio. + def attachment_service_name + record.name.to_sym + end + end + end +end + +ActiveStorage::Attached::Changes::CreateOne.include ActiveStorage::Attached::Changes::CreateOneDecorator From bfb8e95599762f73cc5c8b093932d1750b55c5ea Mon Sep 17 00:00:00 2001 From: f Date: Sun, 6 Mar 2022 12:51:06 -0300 Subject: [PATCH 034/164] obtener el nombre de archivo desde la key esto agrega una query pero permite no tener que agregar una ruta falsa --- .../active_storage/service/jekyll_service.rb | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb index 20dda523..02316e65 100644 --- a/app/lib/active_storage/service/jekyll_service.rb +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -10,8 +10,6 @@ module ActiveStorage # implementemos IPFS) para poder transferir los archivos junto con el # sitio. class JekyllService < Service::DiskService - BLOB_NAME = 'blob' - # Genera un servicio para un sitio determinado def self.build_for_site(site:) new(root: File.join(site.path, 'public'), public: true).tap do |js| @@ -19,16 +17,7 @@ module ActiveStorage end end - # Para poder guardar el archivo con el nombre original pero poder - # recuperarlo durante el download, luego de subirlo le cambiamos el - # nombre y creamos un link simbólico a un nombre conocido. - def upload(key, io, checksum: nil, **options) - super.tap do - path = path_for(key) - filename = options[:filename].to_s - FileUtils.mv path, path.sub(/#{BLOB_NAME}\z/, filename) - FileUtils.ln_s filename, path end end @@ -41,9 +30,17 @@ module ActiveStorage key end + # Obtiene el nombre de archivo para esta key + # + # @param :key [String] + # @return [String] + def filename_for(key) + @filename_for ||= ActiveStorage::Blob.where(key: key).limit(1).pluck(:filename).first + end + # Crea una ruta para la llave con un nombre conocido. def path_for(key) - File.join root, folder_for(key), BLOB_NAME + File.join root, folder_for(key), filename_for(key) end end end From 85cab49208cbfae13b1876ab5a57495544b4220c Mon Sep 17 00:00:00 2001 From: f Date: Sun, 6 Mar 2022 15:25:15 -0300 Subject: [PATCH 035/164] subir los archivos desde el editor al sitio --- .../direct_uploads_controller_decorator.rb | 18 ++++++++++++++++++ app/controllers/posts_controller.rb | 6 ++++++ .../active_storage/service/jekyll_service.rb | 4 ---- 3 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 app/controllers/active_storage/direct_uploads_controller_decorator.rb diff --git a/app/controllers/active_storage/direct_uploads_controller_decorator.rb b/app/controllers/active_storage/direct_uploads_controller_decorator.rb new file mode 100644 index 00000000..f27c4cfb --- /dev/null +++ b/app/controllers/active_storage/direct_uploads_controller_decorator.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module ActiveStorage + # Modifica la creación de un blob antes de subir el archivo para que + # incluya el JekyllService adecuado. + module DirectUploadsControllerDecorator + extend ActiveSupport::Concern + + included do + def create + blob = ActiveStorage::Blob.create_before_direct_upload!(service_name: session[:service_name], **blob_args) + render json: direct_upload_json(blob) + end + end + end +end + +ActiveStorage::DirectUploadsController.include ActiveStorage::DirectUploadsControllerDecorator diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index dbdd4d0a..2aff9ac9 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -6,6 +6,7 @@ class PostsController < ApplicationController rescue_from Pundit::NilPolicyError, with: :page_not_found before_action :authenticate_usuarie! + before_action :service_for_direct_upload, only: %i[new edit] # TODO: Traer los comunes desde ApplicationController breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path @@ -166,4 +167,9 @@ class PostsController < ApplicationController def post @post ||= site.posts(lang: locale).find(params[:post_id] || params[:id]) end + + # Recuerda el nombre del servicio de subida de archivos + def service_for_direct_upload + session[:service_name] = site.name.to_sym + end end diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb index 02316e65..601b6f2f 100644 --- a/app/lib/active_storage/service/jekyll_service.rb +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -17,10 +17,6 @@ module ActiveStorage end end - - end - end - # Mantener retrocompatibilidad con cómo gestionamos los archivos # subidos hasta ahora. # From c13c021fe92d749cf4a98673bc3cc504c6e90d78 Mon Sep 17 00:00:00 2001 From: f Date: Sun, 6 Mar 2022 15:28:25 -0300 Subject: [PATCH 036/164] asignar el nombre de archivo en la subida directa --- .../active_storage/service/jekyll_service.rb | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb index 601b6f2f..a2e04336 100644 --- a/app/lib/active_storage/service/jekyll_service.rb +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -17,6 +17,32 @@ module ActiveStorage end end + # Lo mismo que en DiskService agregando el nombre de archivo en la + # firma. Esto permite que luego podamos guardar el archivo donde + # corresponde. + def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) + instrument :url, key: key do |payload| + verified_token_with_expiration = ActiveStorage.verifier.generate( + { + key: key, + content_type: content_type, + content_length: content_length, + checksum: checksum, + service_name: name, + filename: filename_for(key) + }, + expires_in: expires_in, + purpose: :blob_token + ) + + generated_url = url_helpers.update_rails_disk_service_url(verified_token_with_expiration, host: current_host) + + payload[:url] = generated_url + + generated_url + end + end + # Mantener retrocompatibilidad con cómo gestionamos los archivos # subidos hasta ahora. # From 211fb308e39ddd1a380b711e98d240c59e9576d6 Mon Sep 17 00:00:00 2001 From: f Date: Sun, 6 Mar 2022 15:59:57 -0300 Subject: [PATCH 037/164] asignar archivos subidos desde el editor al sitio --- .../disk_controller_decorator.rb | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 app/controllers/active_storage/disk_controller_decorator.rb diff --git a/app/controllers/active_storage/disk_controller_decorator.rb b/app/controllers/active_storage/disk_controller_decorator.rb new file mode 100644 index 00000000..14366a15 --- /dev/null +++ b/app/controllers/active_storage/disk_controller_decorator.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module ActiveStorage + # Modificar {DiskController} para poder asociar el blob a un sitio + module DiskControllerDecorator + extend ActiveSupport::Concern + + included do + # Asociar el archivo subido al sitio correspondiente. Cada sitio + # tiene su propio servicio de subida de archivos. + def update + if (token = decode_verified_token) + if acceptable_content?(token) + named_disk_service(token[:service_name]).upload token[:key], request.body, checksum: token[:checksum] + + blob = ActiveStorage::Blob.find_by_key token[:key] + site = Site.find_by_name token[:service_name] + + site.static_files.attach(blob) + else + head :unprocessable_entity + end + else + head :not_found + end + rescue ActiveStorage::IntegrityError + head :unprocessable_entity + end + end + end +end + +ActiveStorage::DiskController.include ActiveStorage::DiskControllerDecorator From 61622a4c416b73288cc897b89b8721fe3bb700c9 Mon Sep 17 00:00:00 2001 From: f Date: Sun, 6 Mar 2022 16:00:21 -0300 Subject: [PATCH 038/164] =?UTF-8?q?documentaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/lib/active_storage/service/jekyll_service.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb index a2e04336..3b1db7ec 100644 --- a/app/lib/active_storage/service/jekyll_service.rb +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -11,6 +11,9 @@ module ActiveStorage # sitio. class JekyllService < Service::DiskService # Genera un servicio para un sitio determinado + # + # @param :site [Site] + # @return [ActiveStorage::Service::JekyllService] def self.build_for_site(site:) new(root: File.join(site.path, 'public'), public: true).tap do |js| js.name = site.name.to_sym @@ -20,6 +23,13 @@ module ActiveStorage # Lo mismo que en DiskService agregando el nombre de archivo en la # firma. Esto permite que luego podamos guardar el archivo donde # corresponde. + # + # @param :key [String] + # @param :expires_in [Integer] + # @param :content_type [String] + # @param :content_length [Integer] + # @param :checksum [String] + # @return [String] def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) instrument :url, key: key do |payload| verified_token_with_expiration = ActiveStorage.verifier.generate( @@ -61,6 +71,9 @@ module ActiveStorage end # Crea una ruta para la llave con un nombre conocido. + # + # @param :key [String] + # @return [String] def path_for(key) File.join root, folder_for(key), filename_for(key) end From 6c9288b03bd64f907760d63a5c0ddd638b9ea74f Mon Sep 17 00:00:00 2001 From: f Date: Sun, 6 Mar 2022 16:01:16 -0300 Subject: [PATCH 039/164] =?UTF-8?q?deprecar=20la=20migraci=C3=B3n=20de=20a?= =?UTF-8?q?rchivos=20est=C3=A1ticos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit siempre fue super lenta y falible --- app/models/site.rb | 7 +--- app/models/site/static_file_migration.rb | 52 ------------------------ 2 files changed, 1 insertion(+), 58 deletions(-) delete mode 100644 app/models/site/static_file_migration.rb diff --git a/app/models/site.rb b/app/models/site.rb index 5b78d625..7d4875e5 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -57,7 +57,7 @@ class Site < ApplicationRecord # Carga el sitio Jekyll una vez que se inicializa el modelo o después # de crearlo after_initialize :load_jekyll - after_create :load_jekyll, :static_file_migration! + after_create :load_jekyll # Cambiar el nombre del directorio before_update :update_name! before_save :add_private_key_if_missing! @@ -474,11 +474,6 @@ class Site < ApplicationRecord config.hostname = hostname end - # Migra los archivos a Sutty - def static_file_migration! - Site::StaticFileMigration.new(site: self).migrate! - end - # Valida si el sitio tiene al menos una forma de alojamiento asociada # y es la local # diff --git a/app/models/site/static_file_migration.rb b/app/models/site/static_file_migration.rb deleted file mode 100644 index 36a882bf..00000000 --- a/app/models/site/static_file_migration.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -class Site - # Obtiene todos los archivos relacionados en artículos del sitio y los - # sube a Sutty. - class StaticFileMigration - # Tipos de metadatos que contienen archivos - STATIC_TYPES = %i[file image].freeze - - attr_reader :site - - def initialize(site:) - @site = site - end - - def migrate! - modified = site.docs.map do |doc| - next unless STATIC_TYPES.map do |field| - next unless doc.attribute? field - next unless doc[field].path? - next unless doc[field].static_file - - true - end.any? - - log.write "#{doc.path.relative};no se pudo guardar\n" unless doc.save(validate: false) - - doc.path.absolute - end.compact - - log.close - - return if modified.empty? - - # TODO: Hacer la migración desde el servicio de creación de sitios? - site.repository.commit(file: modified, - message: I18n.t('sites.static_file_migration'), - usuarie: author) - end - - private - - def author - @author ||= GitAuthor.new email: "sutty@#{Site.domain}", - name: 'Sutty' - end - - def log - @log ||= File.open(File.join(site.path, 'migration.csv'), 'w') - end - end -end From f7d8a3ecf9e0142292ea819e7ab94c6b6b096a56 Mon Sep 17 00:00:00 2001 From: f Date: Sun, 6 Mar 2022 17:07:37 -0300 Subject: [PATCH 040/164] fixup! obtener el nombre de archivo desde la key --- app/lib/active_storage/service/jekyll_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb index 3b1db7ec..92b26e0e 100644 --- a/app/lib/active_storage/service/jekyll_service.rb +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -67,7 +67,7 @@ module ActiveStorage # @param :key [String] # @return [String] def filename_for(key) - @filename_for ||= ActiveStorage::Blob.where(key: key).limit(1).pluck(:filename).first + ActiveStorage::Blob.where(key: key).limit(1).pluck(:filename).first end # Crea una ruta para la llave con un nombre conocido. From 81cbafef66e31ba60ac6bf396adc88696d85f142 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 7 Mar 2022 20:52:33 -0300 Subject: [PATCH 041/164] validaciones --- app/models/metadata_file.rb | 2 +- app/models/metadata_image.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 1c859481..5be7f84c 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -19,7 +19,7 @@ class MetadataFile < MetadataTemplate errors << I18n.t("metadata.#{type}.site_invalid") if site.invalid? errors << I18n.t("metadata.#{type}.path_required") if path_missing? errors << I18n.t("metadata.#{type}.no_file_for_description") if no_file_for_description? - errors << I18n.t("metadata.#{type}.attachment_missing") unless static_file + errors << I18n.t("metadata.#{type}.attachment_missing") if path? && !static_file errors.compact! errors.empty? diff --git a/app/models/metadata_image.rb b/app/models/metadata_image.rb index f86c5c26..85ee062a 100644 --- a/app/models/metadata_image.rb +++ b/app/models/metadata_image.rb @@ -5,7 +5,7 @@ class MetadataImage < MetadataFile def validate super - errors << I18n.t('metadata.image.not_an_image') unless image? + errors << I18n.t('metadata.image.not_an_image') unless path? && image? errors.compact! errors.empty? From 7de96a4581a1d1483628c3da11f99e1a4e454ec0 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 8 Mar 2022 13:10:34 -0300 Subject: [PATCH 042/164] =?UTF-8?q?usar=20la=20ruta=20absoluta=20tambi?= =?UTF-8?q?=C3=A9n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit en producción usamos links simbólicos entre directorios y sin resolver la ubicación real de los sitios estábamos generando rutas erróneas. --- app/models/metadata_file.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 5be7f84c..31becc64 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -119,7 +119,7 @@ class MetadataFile < MetadataTemplate end def relative_destination_path_with_filename - destination_path_with_filename.relative_path_from(site.path) + destination_path_with_filename.relative_path_from(Pathname.new(site.path).realpath) end def static_file_path From 79974b710497c2db0498de7f9f4375f2802711c9 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 8 Mar 2022 13:21:13 -0300 Subject: [PATCH 043/164] =?UTF-8?q?validar=20im=C3=A1genes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/metadata_image.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/metadata_image.rb b/app/models/metadata_image.rb index 85ee062a..fc61a4d7 100644 --- a/app/models/metadata_image.rb +++ b/app/models/metadata_image.rb @@ -5,7 +5,7 @@ class MetadataImage < MetadataFile def validate super - errors << I18n.t('metadata.image.not_an_image') unless path? && image? + errors << I18n.t('metadata.image.not_an_image') if path? && !image? errors.compact! errors.empty? From d4d1acc8d91e5cf23db4fa056a5c0520d7179c43 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 9 Mar 2022 19:32:53 -0300 Subject: [PATCH 044/164] retrocompatibilidad los archivos subidos como archivos locales ya estaban copiados al sitio --- app/models/metadata_file.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 31becc64..0f50d643 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -123,7 +123,12 @@ class MetadataFile < MetadataTemplate end def static_file_path - static_file.blob.service.path_for(static_file.key) + case static_file.blob.service.name + when :local + File.join(site.path, 'public', static_file.key, static_file.filename.to_s) + else + static_file.blob.service.path_for(static_file.key) + end end # No hay archivo pero se lo describió From 263394baef931995b78e4a962a6bebfbc5f76b27 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 18 Mar 2022 15:42:39 -0300 Subject: [PATCH 045/164] no fallar al serializar Site#everything_of el problema era que los archivos subidos no se pueden marshalear y estamos usando MetadataTemplate#values para poder cachearlos. closes #1315 closes #1316 closes #1842 closes #2368 closes #2369 closes #2370 closes #2371 closes #2372 closes #2373 closes #2393 closes #2396 closes #2922 closes #3347 closes #3350 closes #3353 closes #3356 closes #3437 closes #3486 closes #3487 closes #3488 closes #3551 closes #3554 closes #4229 closes #4256 closes #4259 closes #4262 closes #4265 closes #4553 closes #4556 closes #4603 closes #4606 closes #4647 closes #4759 closes #4767 closes #4787 closes #4794 closes #4833 closes #4835 closes #4836 closes #4875 closes #4876 closes #4877 closes #4879 closes #4881 closes #4882 closes #4883 closes #4884 closes #4885 closes #4886 closes #4887 closes #4888 closes #4889 closes #4890 closes #4891 closes #4892 closes #4906 closes #4907 closes #4908 closes #4976 closes #4977 closes #4978 closes #4979 closes #4980 closes #4981 closes #4982 closes #4983 closes #4984 closes #4985 closes #4986 closes #4987 --- app/models/metadata_file.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 0f50d643..2c1a6d59 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -13,6 +13,11 @@ class MetadataFile < MetadataTemplate value == default_value end + # No hay valores sugeridos para archivos subidos. + # + # XXX: Esto ayuda a deserializar en {Site#everything_of} + def values; end + def validate super From c5896a63cfdd48128592087aac35aaf1a6523df1 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 22 Mar 2022 20:15:59 -0300 Subject: [PATCH 046/164] resiliencia closes #5061 closes #5060 closes #5058 closes #5057 closes #5056 closes #5054 closes #5053 closes #5051 closes #5050 closes #5048 closes #5047 closes #5045 closes #5042 closes #5040 closes #5038 closes #5036 closes #5034 closes #5033 closes #5032 closes #5030 closes #4946 closes #3856 closes #3562 closes #3558 closes #3557 closes #2707 closes #2706 closes #2705 closes #2703 closes #2702 closes #2297 closes #2296 closes #2295 closes #1970 closes #1969 closes #1768 --- app/models/metadata_file.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 2c1a6d59..eca87478 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -121,6 +121,13 @@ class MetadataFile < MetadataTemplate # @return [Pathname] def destination_path_with_filename destination_path.realpath + # Si el archivo no llegara a existir, en lugar de hacer fallar todo, + # devolvemos la ruta original, que puede ser el archivo que no existe + # o vacía si se está subiendo uno. + rescue Errno::ENOENT => e + ExceptionNotifier.notify_exception(e) + + value['path'] end def relative_destination_path_with_filename From ecb823f40746b4eae0d61cc3acca63ac620881f0 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 23 Mar 2022 11:37:00 -0300 Subject: [PATCH 047/164] al migrar un archivo reutilizar la key --- app/models/metadata_file.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index eca87478..71d3f049 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -78,7 +78,9 @@ class MetadataFile < MetadataTemplate if (blob_id = ActiveStorage::Blob.where(key: key_from_path).pluck(:id).first) site.static_files.find_by(blob_id: blob_id) elsif path? && pathname.exist? && site.static_files.attach(io: pathname.open, filename: pathname.basename) - site.static_files.last + site.static_files.last.tap do |s| + s.blob.update(key: key_from_path) + end end end end From 455070f2ea576775fe28f0bb2d01a6b62d2dc7d0 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 4 Apr 2022 13:56:40 -0300 Subject: [PATCH 048/164] normalizar todas las strings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit convierte la codificación de windows/osx en unicode normalizado. --- app/models/metadata_markdown.rb | 2 +- app/models/metadata_markdown_content.rb | 2 +- app/models/metadata_permalink.rb | 2 +- app/models/metadata_string.rb | 2 +- app/models/metadata_template.rb | 9 ++++++--- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/app/models/metadata_markdown.rb b/app/models/metadata_markdown.rb index 1e8b4fc8..7816ec33 100644 --- a/app/models/metadata_markdown.rb +++ b/app/models/metadata_markdown.rb @@ -12,6 +12,6 @@ class MetadataMarkdown < MetadataText # markdown y se eliminan autolinks. Mejor es habilitar la generación # SAFE de CommonMark en la configuración del sitio. def sanitize(string) - string + string.unicode_normalize(:nfkc) end end diff --git a/app/models/metadata_markdown_content.rb b/app/models/metadata_markdown_content.rb index 92a1ab21..cb4124db 100644 --- a/app/models/metadata_markdown_content.rb +++ b/app/models/metadata_markdown_content.rb @@ -25,6 +25,6 @@ class MetadataMarkdownContent < MetadataText # markdown y se eliminan autolinks. Mejor es deshabilitar la # generación SAFE de CommonMark en la configuración del sitio. def sanitize(string) - string.tr("\r", '') + string.tr("\r", '').unicode_normalize(:nfkc) end end diff --git a/app/models/metadata_permalink.rb b/app/models/metadata_permalink.rb index 59b68461..9b0c063c 100644 --- a/app/models/metadata_permalink.rb +++ b/app/models/metadata_permalink.rb @@ -19,7 +19,7 @@ class MetadataPermalink < MetadataString # puntos suspensivos, la primera / para que siempre sea relativa y # agregamos una / al final si la ruta no tiene extensión. def sanitize(value) - value = value.strip.gsub('..', '/').gsub('./', '').squeeze('/') + value = value.strip.unicode_normalize(:nfkc).gsub('..', '/').gsub('./', '').squeeze('/') value = value[1..-1] if value.start_with? '/' value += '/' if File.extname(value).blank? diff --git a/app/models/metadata_string.rb b/app/models/metadata_string.rb index 95aac4d4..28bfe82a 100644 --- a/app/models/metadata_string.rb +++ b/app/models/metadata_string.rb @@ -17,7 +17,7 @@ class MetadataString < MetadataTemplate def sanitize(string) return '' if string.blank? - sanitizer.sanitize(string.strip, + sanitizer.sanitize(string.strip.unicode_normalize(:nfkc), tags: [], attributes: []).strip.html_safe end diff --git a/app/models/metadata_template.rb b/app/models/metadata_template.rb index 5baa7a4a..a72f8e83 100644 --- a/app/models/metadata_template.rb +++ b/app/models/metadata_template.rb @@ -184,9 +184,12 @@ MetadataTemplate = Struct.new(:site, :document, :name, :label, :type, return if string.nil? return string unless string.is_a? String - sanitizer.sanitize(string.tr("\r", ''), - tags: allowed_tags, - attributes: allowed_attributes).strip.html_safe + sanitizer + .sanitize(string.tr("\r", '').unicode_normalize(:nfkc), + tags: allowed_tags, + attributes: allowed_attributes) + .strip + .html_safe end def sanitizer From 30bc14d83e790c6540a6fb19e8afa8184eef6c37 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 4 Apr 2022 14:27:00 -0300 Subject: [PATCH 049/164] normalizar los nombres de archivo enviados por subida directa --- .../direct_uploads_controller_decorator.rb | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/controllers/active_storage/direct_uploads_controller_decorator.rb b/app/controllers/active_storage/direct_uploads_controller_decorator.rb index f27c4cfb..3052f974 100644 --- a/app/controllers/active_storage/direct_uploads_controller_decorator.rb +++ b/app/controllers/active_storage/direct_uploads_controller_decorator.rb @@ -11,6 +11,17 @@ module ActiveStorage blob = ActiveStorage::Blob.create_before_direct_upload!(service_name: session[:service_name], **blob_args) render json: direct_upload_json(blob) end + + private + + # Normalizar los caracteres unicode en los nombres de archivos + # para que puedan propagarse correctamente a través de todo el + # stack. + def blob_args + params.require(:blob).permit(:filename, :byte_size, :checksum, :content_type, metadata: {}).to_h.symbolize_keys.tap do |ba| + ba[:filename] = ba[:filename].unicode_normalize(:nfkc) + end + end end end end From 4d6a26d67178ec90e4b8f580889cee528f74d195 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 4 Apr 2022 14:32:58 -0300 Subject: [PATCH 050/164] normalizar todos los nombres de archivos subidos --- .../http/uploaded_file_decorator.rb | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 app/lib/action_dispatch/http/uploaded_file_decorator.rb diff --git a/app/lib/action_dispatch/http/uploaded_file_decorator.rb b/app/lib/action_dispatch/http/uploaded_file_decorator.rb new file mode 100644 index 00000000..c171c81c --- /dev/null +++ b/app/lib/action_dispatch/http/uploaded_file_decorator.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module ActionDispatch + module Http + # Normaliza los nombres de archivo para que se propaguen + # correctamente a través de todo el stack. + module UploadedFileDecorator + extend ActiveSupport::Concern + + included do + # Devolver el nombre de archivo con caracteres unicode + # normalizados + def original_filename + @original_filename.unicode_normalize(:nfkc) + end + end + end + end +end + +ActionDispatch::Http::UploadedFile.include ActionDispatch::Http::UploadedFileDecorator From accb559f014ccc287ffcc5e6f15ce5ddae918a53 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 16:39:56 -0300 Subject: [PATCH 051/164] hacer deploys remotos --- app/models/deploy_rsync.rb | 88 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 app/models/deploy_rsync.rb diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb new file mode 100644 index 00000000..40018dbd --- /dev/null +++ b/app/models/deploy_rsync.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +# Sincroniza sitios a servidores remotos usando Rsync. El servidor +# remoto tiene que tener rsync instalado. +class DeployRsync < Deploy + store :values, accessors: %i[flags destination host_keys], coder: JSON + + def deploy + ssh? && rsync + end + + # No se ocupa espacio local + # + # @return [Integer] + def size + 0 + end + + private + + # Verificar la conexión SSH implementando Trust On First Use + # + # TODO: Medir el tiempo que tarda en iniciarse la conexión + # + # @return [Boolean] + def ssh? + user, host = user_host + ssh_available = false + + Net::SSH.start(host, user, verify_host_key: tofu) do |ssh| + if values[:host_keys].blank? + # Guardar las llaves que se encontraron en la primera conexión + values[:host_keys] = ssh.transport.host_keys.map do |host_key| + "#{host_key.ssh_type} #{host_key.fingerprint}" + end + + ssh_available = save + else + ssh_available = true + end + end + + ssh_available + rescue Exception => e + ExceptionNotifier.notify_exception(e, data: { site: site.id, hostname: host, user: user }) + + false + end + + # Confiar en la primera llave que encontremos, fallar si cambian + # + # @return [Symbol] + def tofu + values[:host_keys].present? ? :always : :accept_new + end + + # Devuelve el par user host + # + # @return [Array] + def user_host + destination.split(':', 2).first.split('@', 2).tap do |d| + next unless d.size == 1 + + d.insert(0, nil) + end + end + + # Sincroniza hacia el directorio remoto, usando las flags opcionales. + # + # @return [Boolean] + def rsync + run %(rsync -av #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + end + + # El origen es el destino de la compilación + # + # @return [String] + def source + site.deploys.find_by(type: 'DeployLocal').destination + end + + # Devolver el destino o lanzar un error si no está configurado + def destination + values[:destination].tap do |_d| + raise ArgumentError, 'destination no está configurado' + end + end +end From f10b65173a9542500e3d206a2167fb3524b59ed0 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 17:05:51 -0300 Subject: [PATCH 052/164] fixup! hacer deploys remotos --- app/models/deploy_rsync.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 40018dbd..9e855f57 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -81,8 +81,8 @@ class DeployRsync < Deploy # Devolver el destino o lanzar un error si no está configurado def destination - values[:destination].tap do |_d| - raise ArgumentError, 'destination no está configurado' + values[:destination].tap do |d| + raise(ArgumentError, 'destination no está configurado') if d.blank? end end end From 6fcdeb52f381bbde75b5156671f228016209f6ea Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 17:23:42 -0300 Subject: [PATCH 053/164] entorno --- app/models/deploy_rsync.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 9e855f57..4191b20d 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -47,6 +47,14 @@ class DeployRsync < Deploy false end + def env + { + 'HOME' => home_dir, + 'PATH' => '/usr/bin', + 'LANG' => ENV['LANG'] + } + end + # Confiar en la primera llave que encontremos, fallar si cambian # # @return [Symbol] From ea198f185b655a353755b097cdbc6c264378db34 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:07:45 -0300 Subject: [PATCH 054/164] bajar el timeout para no bloquear el deploy --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 4191b20d..2387366a 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -27,7 +27,7 @@ class DeployRsync < Deploy user, host = user_host ssh_available = false - Net::SSH.start(host, user, verify_host_key: tofu) do |ssh| + Net::SSH.start(host, user, verify_host_key: tofu, timeout: 5) do |ssh| if values[:host_keys].blank? # Guardar las llaves que se encontraron en la primera conexión values[:host_keys] = ssh.transport.host_keys.map do |host_key| From c49c63f77661413a208a47bd7a80f7b4b8d49535 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:11:21 -0300 Subject: [PATCH 055/164] timeout de rsync --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 2387366a..5d360c4c 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -77,7 +77,7 @@ class DeployRsync < Deploy # # @return [Boolean] def rsync - run %(rsync -av #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -av --timeout=5 #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) end # El origen es el destino de la compilación From 8edc9b460af993ad4c52b1192962cb548c39a82a Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:17:55 -0300 Subject: [PATCH 056/164] obtener listado de nodos de un lugar central --- app/models/sutty.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 app/models/sutty.rb diff --git a/app/models/sutty.rb b/app/models/sutty.rb new file mode 100644 index 00000000..7ec8432c --- /dev/null +++ b/app/models/sutty.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# Configuración general de Sutty +class Sutty + extend self + + # Los nodos son otros servidores de Sutty hacia los que se sincronizan + # sitios. + # + # @return [Array] + def nodes + @nodes ||= ENV.fetch('SUTTY_NODES', '').split(',') + end +end From dcbd1c02ac32c1b183d6c90255da693ad67bbc9e Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:26:41 -0300 Subject: [PATCH 057/164] =?UTF-8?q?el=20m=C3=A9todo=20es=20p=C3=BAblico?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_rsync.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 5d360c4c..50719cd1 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -16,6 +16,13 @@ class DeployRsync < Deploy 0 end + # Devolver el destino o lanzar un error si no está configurado + def destination + values[:destination].tap do |d| + raise(ArgumentError, 'destination no está configurado') if d.blank? + end + end + private # Verificar la conexión SSH implementando Trust On First Use @@ -86,11 +93,4 @@ class DeployRsync < Deploy def source site.deploys.find_by(type: 'DeployLocal').destination end - - # Devolver el destino o lanzar un error si no está configurado - def destination - values[:destination].tap do |d| - raise(ArgumentError, 'destination no está configurado') if d.blank? - end - end end From 5fac827ca09fd46231490c59f845b6ec6a2e6121 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:27:15 -0300 Subject: [PATCH 058/164] registrar los cambios --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 50719cd1..7733406e 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -84,7 +84,7 @@ class DeployRsync < Deploy # # @return [Boolean] def rsync - run %(rsync -av --timeout=5 #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -avi --timeout=5 #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) end # El origen es el destino de la compilación From 20915443d84d7ddb50ef8417d957d2a2dbb2dbcd Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:27:52 -0300 Subject: [PATCH 059/164] deprecar las flags por ahora --- app/models/deploy_rsync.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 7733406e..12f4ce0e 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -3,7 +3,7 @@ # Sincroniza sitios a servidores remotos usando Rsync. El servidor # remoto tiene que tener rsync instalado. class DeployRsync < Deploy - store :values, accessors: %i[flags destination host_keys], coder: JSON + store :values, accessors: %i[destination host_keys], coder: JSON def deploy ssh? && rsync @@ -80,11 +80,11 @@ class DeployRsync < Deploy end end - # Sincroniza hacia el directorio remoto, usando las flags opcionales. + # Sincroniza hacia el directorio remoto # # @return [Boolean] def rsync - run %(rsync -avi --timeout=5 #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -avi --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) end # El origen es el destino de la compilación From a5e90257a30566ecd00fc36a63bf8fcb5e789528 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:38:03 -0300 Subject: [PATCH 060/164] generar un deploy rsync para el sitio con todos los nodos --- app/services/site_service.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 5e2fc706..2ffb8cdd 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -144,4 +144,11 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do PostService.new(site: site, usuarie: usuarie, post: post, params: params).update end + + # Crea los deploys necesarios para sincronizar a otros nodos de Sutty + def sync_nodes + Sutty.nodes.each do |node| + site.deploys.build(type: 'DeployRsync', destination: "sutty@#{node}:#{site.hostname}") + end + end end From 10d950689b614f6ca2a99ace3cf14dce44b3a037 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:39:52 -0300 Subject: [PATCH 061/164] crear rsyncs al crear el sitio --- app/services/site_service.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 2ffb8cdd..89c5796d 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -9,6 +9,7 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do self.site = Site.new params add_role temporal: false, rol: 'usuarie' + sync_nodes I18n.with_locale(usuarie&.lang&.to_sym || I18n.default_locale) do site.save && From a0134cc052578e2ec66025b3d51b2879fc144217 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 19:05:50 -0300 Subject: [PATCH 062/164] =?UTF-8?q?ya=20existe=20un=20lugar=20donde=20guar?= =?UTF-8?q?dar=20la=20configuraci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/sutty.rb | 14 -------------- app/services/site_service.rb | 2 +- config/application.rb | 4 ++++ 3 files changed, 5 insertions(+), 15 deletions(-) delete mode 100644 app/models/sutty.rb diff --git a/app/models/sutty.rb b/app/models/sutty.rb deleted file mode 100644 index 7ec8432c..00000000 --- a/app/models/sutty.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -# Configuración general de Sutty -class Sutty - extend self - - # Los nodos son otros servidores de Sutty hacia los que se sincronizan - # sitios. - # - # @return [Array] - def nodes - @nodes ||= ENV.fetch('SUTTY_NODES', '').split(',') - end -end diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 89c5796d..22423bb8 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -148,7 +148,7 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do # Crea los deploys necesarios para sincronizar a otros nodos de Sutty def sync_nodes - Sutty.nodes.each do |node| + Rails.application.nodes.each do |node| site.deploys.build(type: 'DeployRsync', destination: "sutty@#{node}:#{site.hostname}") end end diff --git a/config/application.rb b/config/application.rb index 7326ae0f..bc948936 100644 --- a/config/application.rb +++ b/config/application.rb @@ -49,5 +49,9 @@ module Sutty EmailAddress::Config.error_messages translations.transform_keys(&:to_s), locale.to_s end end + + def nodes + @nodes ||= ENV.fetch('SUTTY_NODES', '').split(',') + end end end From aa0c359554917ef93547d3446e55ac73394587a8 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 19:06:11 -0300 Subject: [PATCH 063/164] crear un rsync por cada sitio existente --- .../20220406211042_add_deploy_rsync_to_sites.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 db/migrate/20220406211042_add_deploy_rsync_to_sites.rb diff --git a/db/migrate/20220406211042_add_deploy_rsync_to_sites.rb b/db/migrate/20220406211042_add_deploy_rsync_to_sites.rb new file mode 100644 index 00000000..92b6f17b --- /dev/null +++ b/db/migrate/20220406211042_add_deploy_rsync_to_sites.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# Agrega un DeployRsync hacia los servidores alternativos para cada +# sitio +class AddDeployRsyncToSites < ActiveRecord::Migration[6.1] + def up + Site.find_each do |site| + SiteService.new(site: site).send :sync_nodes + site.save + end + end + + def down + DeployRsync.destroy_all + end +end From f51c051af81da89345e66f137be419b00800ebe5 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 19:35:35 -0300 Subject: [PATCH 064/164] mensajes --- config/locales/en.yml | 4 ++++ config/locales/es.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/config/locales/en.yml b/config/locales/en.yml index b814796d..647f3ee8 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -100,6 +100,10 @@ en: title: Alternative domain name success: Success! error: Error + deploy_rsync: + title: Synchronize to backup server + success: Success! + error: Error help: You can contact us by replying to this e-mail maintenance_mailer: notice: diff --git a/config/locales/es.yml b/config/locales/es.yml index a6fbd407..1ccf5047 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -100,6 +100,10 @@ es: title: Dominio alternativo success: ¡Éxito! error: Hubo un error + deploy_rsync: + title: Sincronizar al servidor alternativo + success: ¡Éxito! + error: Hubo un error help: Por cualquier duda, responde este correo para contactarte con nosotres. maintenance_mailer: notice: From 8c9bd6aa88c9c584896c9109d4c04aad5d9b1cfd Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:31:49 -0300 Subject: [PATCH 065/164] por ahora no es configurable --- app/views/deploys/_deploy_rsync.haml | 1 + 1 file changed, 1 insertion(+) create mode 100644 app/views/deploys/_deploy_rsync.haml diff --git a/app/views/deploys/_deploy_rsync.haml b/app/views/deploys/_deploy_rsync.haml new file mode 100644 index 00000000..0aab9802 --- /dev/null +++ b/app/views/deploys/_deploy_rsync.haml @@ -0,0 +1 @@ +-# nada From 35e41b729ad35e86b1b4d4ce63ba2d4dc0ad97d6 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:44:46 -0300 Subject: [PATCH 066/164] el espacio ocupado es el mismo que el local --- app/models/deploy_rsync.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 12f4ce0e..c19d279b 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -9,11 +9,11 @@ class DeployRsync < Deploy ssh? && rsync end - # No se ocupa espacio local + # El espacio remoto es el mismo que el local # # @return [Integer] def size - 0 + deploy_local.build_stats.last.size end # Devolver el destino o lanzar un error si no está configurado @@ -91,6 +91,10 @@ class DeployRsync < Deploy # # @return [String] def source - site.deploys.find_by(type: 'DeployLocal').destination + deploy_local.destination + end + + def deploy_local + @deploy_local ||= site.deploys.find_by(type: 'DeployLocal') end end From ea25d27f240cacad4fac4a1bf97455aa03c4c8f5 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:53:45 -0300 Subject: [PATCH 067/164] =?UTF-8?q?calcular=20el=20tama=C3=B1o=20a=20parti?= =?UTF-8?q?r=20de=20deploy=20local?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index c19d279b..b2ffba7c 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -13,7 +13,7 @@ class DeployRsync < Deploy # # @return [Integer] def size - deploy_local.build_stats.last.size + deploy_local.size end # Devolver el destino o lanzar un error si no está configurado From bb7c50e24b4fb9a9ce16666d53a77fc666a22506 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:54:35 -0300 Subject: [PATCH 068/164] calcular el espacio una sola vez --- app/models/deploy_local.rb | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index 4fa588f5..1b661059 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -28,15 +28,17 @@ class DeployLocal < Deploy # Obtener el tamaño de todos los archivos y directorios (los # directorios son archivos :) def size - paths = [destination, File.join(destination, '**', '**')] + @size ||= begin + paths = [destination, File.join(destination, '**', '**')] - Dir.glob(paths).map do |file| - if File.symlink? file - 0 - else - File.size(file) - end - end.inject(:+) + Dir.glob(paths).map do |file| + if File.symlink? file + 0 + else + File.size(file) + end + end.inject(:+) + end end def destination From 13561a5f717af796371a244daf11b2e7d3d029d7 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 19 Apr 2022 10:07:10 -0300 Subject: [PATCH 069/164] =?UTF-8?q?no=20generar=20errores=20si=20se=20env?= =?UTF-8?q?=C3=ADa=20el=20reporte=20incompleto?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closes #5136 --- app/controllers/api/v1/csp_reports_controller.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/v1/csp_reports_controller.rb b/app/controllers/api/v1/csp_reports_controller.rb index bc6cfae0..f1d7a376 100644 --- a/app/controllers/api/v1/csp_reports_controller.rb +++ b/app/controllers/api/v1/csp_reports_controller.rb @@ -6,6 +6,9 @@ module Api class CspReportsController < BaseController skip_forgery_protection + # No queremos indicar que algo salió mal + rescue_from ActionController::ParameterMissing, with: :csp_report_created + # Crea un reporte de CSP intercambiando los guiones medios por # bajos # @@ -18,7 +21,7 @@ module Api csp.id = SecureRandom.uuid csp.save - render json: {}, status: :created + csp_report_created end private @@ -39,6 +42,10 @@ module Api :'column-number', :'source-file') end + + def csp_report_created + render json: {}, status: :created + end end end end From 52f446fcf1e8fc9c31a1ee9ef1028f320bb24304 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 19 Apr 2022 10:57:01 -0300 Subject: [PATCH 070/164] refactorizacion --- app/controllers/api/v1/csp_reports_controller.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/v1/csp_reports_controller.rb b/app/controllers/api/v1/csp_reports_controller.rb index f1d7a376..ea186729 100644 --- a/app/controllers/api/v1/csp_reports_controller.rb +++ b/app/controllers/api/v1/csp_reports_controller.rb @@ -14,9 +14,9 @@ module Api # # TODO: Aplicar rate_limit def create - csp = CspReport.new(csp_report_params.to_h.map do |k, v| - [k.tr('-', '_'), v] - end.to_h) + csp = CspReport.new(csp_report_params.to_h.transform_keys do |k| + k.tr('-', '_') + end) csp.id = SecureRandom.uuid csp.save From ed3f15391982e93c2902c2a33582c0ce063c5c35 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 15:06:00 -0300 Subject: [PATCH 071/164] =?UTF-8?q?es=20una=20tarea=20peri=C3=B3dica?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 2aa8d702..16f04738 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -2,7 +2,7 @@ # Genera resúmenes de información para poder mostrar estadísticas y se # corre regularmente a sí misma. -class StatCollectionJob < ApplicationJob +class StatCollectionJob < PeriodicJob STAT_NAME = 'stat_collection_job' def perform(site_id:, once: true) From 30861d240ea08e415c709c639d9b8cc87495b239 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 15:06:17 -0300 Subject: [PATCH 072/164] =?UTF-8?q?aplicar=20la=20operaci=C3=B3n!?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 16f04738..e70544c3 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -44,7 +44,7 @@ class StatCollectionJob < PeriodicJob .where_dimensions(site_id: site.id) .group("dimensions->'site_id'") .rollup(name, interval: interval, update: true) do |rollup| - rollup.try(:operation, :value) + rollup.try(operation, :value) end end From 22742cf058cef5ebf2d562b4e91ee6da2abe833f Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 15:06:55 -0300 Subject: [PATCH 073/164] migraciones faltantes --- db/migrate/20211022224008_add_site_to_stats.rb | 8 ++++++++ db/migrate/20211022225449_add_name_to_stats.rb | 9 +++++++++ 2 files changed, 17 insertions(+) create mode 100644 db/migrate/20211022224008_add_site_to_stats.rb create mode 100644 db/migrate/20211022225449_add_name_to_stats.rb diff --git a/db/migrate/20211022224008_add_site_to_stats.rb b/db/migrate/20211022224008_add_site_to_stats.rb new file mode 100644 index 00000000..db2b43ab --- /dev/null +++ b/db/migrate/20211022224008_add_site_to_stats.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +# La recolección de estadísticas podría pertenecer a un sitio +class AddSiteToStats < ActiveRecord::Migration[6.1] + def change + add_belongs_to :stats, :site, index: true, null: true + end +end diff --git a/db/migrate/20211022225449_add_name_to_stats.rb b/db/migrate/20211022225449_add_name_to_stats.rb new file mode 100644 index 00000000..89a17ee0 --- /dev/null +++ b/db/migrate/20211022225449_add_name_to_stats.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +# Agregarle un nombre a la estadística +class AddNameToStats < ActiveRecord::Migration[6.1] + def change + add_column :stats, :name, :string, null: false + add_index :stats, :name, using: 'hash' + end +end From 9b2194ee950354573fdef604a49cdafb8141d0e7 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 15:24:06 -0300 Subject: [PATCH 074/164] detener la tarea --- app/jobs/periodic_job.rb | 8 ++++++++ app/jobs/uri_collection_job.rb | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/app/jobs/periodic_job.rb b/app/jobs/periodic_job.rb index 8d9453a3..2f60a2b3 100644 --- a/app/jobs/periodic_job.rb +++ b/app/jobs/periodic_job.rb @@ -52,4 +52,12 @@ class PeriodicJob < ApplicationJob def beginning_of_interval @beginning_of_interval ||= last_stat.created_at.try(:"beginning_of_#{starting_interval}") end + + def stop_file + @stop_file ||= Rails.root.join('tmp', self.class.to_s.tableize) + end + + def stop? + File.exist? stop_file + end end diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 9ec333cd..b66a804d 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -21,7 +21,7 @@ class UriCollectionJob < PeriodicJob hostnames.each do |hostname| uris.each do |uri| - return if File.exist? Rails.root.join('tmp', 'uri_collection_job_stop') + return if stop? AccessLog.where(host: hostname, uri: uri) .where('created_at >= ?', beginning_of_interval) From ef32a6d7f6e94b266f19c35ea565db063eb3ced7 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 15:30:03 -0300 Subject: [PATCH 075/164] schema --- db/schema.rb | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/db/schema.rb b/db/schema.rb index 107e7be7..a395329d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_05_14_165639) do +ActiveRecord::Schema.define(version: 2021_10_22_225449) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -64,6 +64,7 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do t.string "remote_user" t.boolean "crawler", default: false t.string "http_referer" + t.datetime "created_at", precision: 6 t.index ["geoip2_data_city_name"], name: "index_access_logs_on_geoip2_data_city_name" t.index ["geoip2_data_country_name"], name: "index_access_logs_on_geoip2_data_country_name" t.index ["host"], name: "index_access_logs_on_host" @@ -303,6 +304,15 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do t.index ["usuarie_id"], name: "index_roles_on_usuarie_id" end + create_table "rollups", force: :cascade do |t| + t.string "name", null: false + t.string "interval", null: false + t.datetime "time", null: false + t.jsonb "dimensions", default: {}, null: false + t.float "value" + t.index ["name", "interval", "time", "dimensions"], name: "index_rollups_on_name_and_interval_and_time_and_dimensions", unique: true + end + create_table "sites", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -324,6 +334,15 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do t.index ["name"], name: "index_sites_on_name", unique: true end + create_table "stats", force: :cascade do |t| + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.bigint "site_id" + t.string "name", null: false + t.index ["name"], name: "index_stats_on_name", using: :hash + t.index ["site_id"], name: "index_stats_on_site_id" + end + create_table "usuaries", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -370,4 +389,10 @@ new.indexed_content := to_tsvector(('pg_catalog.' || new.dictionary)::regconfig, SQL_ACTIONS end + create_trigger("access_logs_before_insert_row_tr", :compatibility => 1). + on("access_logs"). + before(:insert) do + "new.created_at := to_timestamp(new.msec);" + end + end From cd1263a67211d103ccaab46b1bfe89cdc97ed536 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 15:46:24 -0300 Subject: [PATCH 076/164] usar fork por ahora --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 2b304ee0..1e476dde 100644 --- a/Gemfile +++ b/Gemfile @@ -64,7 +64,7 @@ gem 'rails-i18n' gem 'rails_warden' gem 'redis', require: %w[redis redis/connection/hiredis] gem 'redis-rails' -gem 'rollups', git: 'https://github.com/ankane/rollup.git', branch: 'master' +gem 'rollups', git: 'https://github.com/fauno/rollup.git', branch: 'update' gem 'rubyzip' gem 'rugged' gem 'concurrent-ruby-ext' From f96c7aff7d7dde7e3c874fb7f7b78e16a11c198f Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 16:28:32 -0300 Subject: [PATCH 077/164] =?UTF-8?q?volcar=20y=20eliminar=20los=20logs=20de?= =?UTF-8?q?l=20d=C3=ADa=20anterior?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/access_logs | 12 ++++++++++++ monit.conf | 5 +++++ 2 files changed, 17 insertions(+) create mode 100755 bin/access_logs diff --git a/bin/access_logs b/bin/access_logs new file mode 100755 index 00000000..2b4fd99f --- /dev/null +++ b/bin/access_logs @@ -0,0 +1,12 @@ +#!/bin/sh +set -e + +# Volcar y eliminar todos los access logs del día anterior +date=`date +%F` + +psql -h postgresql "${DATABASE:-sutty}" sutty < "/srv/http/_storage/${date}.psql.gz" +begin; +copy (select * from access_logs where created_at < '${date}') to stdout; +delete from access_logs where created_at < '${date}'; +commit; +SQL diff --git a/monit.conf b/monit.conf index f574c56d..27605d7c 100644 --- a/monit.conf +++ b/monit.conf @@ -29,3 +29,8 @@ check program blazer as uid "app" and gid "www-data" every 61 cycles if status != 0 then alert + +check program access_logs + with path "/srv/http/bin/access_logs" as uid "app" and gid "www-data" + every "0 0 * * *" + if status != 0 then alert From 83b9325d0c76e74a5cf525df45eb67b660f0246e Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 17:50:36 -0300 Subject: [PATCH 078/164] =?UTF-8?q?grabar=20la=20duraci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index b66a804d..7ca9a4d9 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -19,9 +19,12 @@ class UriCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id + # Recordar la última vez que se corrió la tarea + stat = site.stats.create! name: STAT_NAME + hostnames.each do |hostname| uris.each do |uri| - return if stop? + next if stop? AccessLog.where(host: hostname, uri: uri) .where('created_at >= ?', beginning_of_interval) @@ -46,8 +49,7 @@ class UriCollectionJob < PeriodicJob end end - # Recordar la última vez que se corrió la tarea - site.stats.create! name: STAT_NAME + stat.touch run_again! unless once end From 0cd71ad779f05923de7f1c94cc3a89e880c9aa35 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 17:54:36 -0300 Subject: [PATCH 079/164] eliminar blazer de las rutas por sitio --- config/routes.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/config/routes.rb b/config/routes.rb index c323f556..8bab18af 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -36,9 +36,6 @@ Rails.application.routes.draw do match '/api/v3/projects/:site_id/notices' => 'api/v1/notices#create', via: %i[post] resources :sites, constraints: { site_id: %r{[^/]+}, id: %r{[^/]+} } do - # Usar Blazer para mostrar estadísticas - mount Blazer::Engine, at: 'stats', as: 'stats' - # Gestionar actualizaciones del sitio get 'pull', to: 'sites#fetch' post 'pull', to: 'sites#merge' From 3edf45e27fcdd51fdf64730bc856fcf021224dc2 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 18:04:51 -0300 Subject: [PATCH 080/164] fixup! usar fork por ahora --- Gemfile.lock | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 8df2d77e..87812726 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,15 +6,6 @@ GIT rails (>= 3.0) rake (>= 0.8.7) -GIT - remote: https://github.com/ankane/rollup.git - revision: 0ab6c603450175eb1004f7793e86486943cb9f72 - branch: master - specs: - rollups (0.1.3) - activesupport (>= 5.1) - groupdate (>= 5.2) - GIT remote: https://github.com/fauno/email_address revision: 536b51f7071b68a55140c0c1726b4cd401d1c04d @@ -24,6 +15,15 @@ GIT netaddr (>= 2.0.4, < 3) simpleidn +GIT + remote: https://github.com/fauno/rollup.git + revision: ddbb345aa57e63b4cfdf7557267efa89ba60caac + branch: update + specs: + rollups (0.1.3) + activesupport (>= 5.1) + groupdate (>= 5.2) + GEM remote: https://gems.sutty.nl/ specs: @@ -214,8 +214,8 @@ GEM ffi (~> 1.0) globalid (0.6.0) activesupport (>= 5.0) - groupdate (5.2.2) - activesupport (>= 5) + groupdate (6.1.0) + activesupport (>= 5.2) hairtrigger (0.2.24) activerecord (>= 5.0, < 7) ruby2ruby (~> 2.4) From bfdd90088f76fb88560f9745218eb60e1c43ceed Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 18:29:18 -0300 Subject: [PATCH 081/164] permisos --- app/models/site_stat.rb | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 app/models/site_stat.rb diff --git a/app/models/site_stat.rb b/app/models/site_stat.rb new file mode 100644 index 00000000..73503aca --- /dev/null +++ b/app/models/site_stat.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +SiteStat = Struct.new(:site) From 0c34613cb579712045696c5d6a213c02491ddf5a Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 18:56:34 -0300 Subject: [PATCH 082/164] encontrar todos los idiomas --- app/jobs/uri_collection_job.rb | 50 ++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 7ca9a4d9..ada61bab 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -13,7 +13,7 @@ class UriCollectionJob < PeriodicJob # Ignoramos imágenes porque suelen ser demasiadas y no aportan a las # estadísticas. - IMAGES = %w[.png .jpg .jpeg .gif .webp].freeze + IMAGES = %w[.png .jpg .jpeg .gif .webp .jfif].freeze STAT_NAME = 'uri_collection_job' def perform(site_id:, once: true) @@ -91,18 +91,46 @@ class UriCollectionJob < PeriodicJob # # @return [Array] def uris - @uris ||= Dir.chdir destination do - (Dir.glob('**/*.html') + Dir.glob('public/**/*').reject do |p| - File.directory? p - end.reject do |p| - p = p.downcase + @uris ||= + locales.map do |locale| + uri = "/#{locale}/".squeeze('/') + dir = File.join(destination, locale) - IMAGES.any? do |i| - p.end_with? i + files(dir).map do |f| + uri + f end - end).map do |uri| - "/#{uri}" - end + end.flatten(2) + end + + # @return [Array] + def locales + @locales ||= ['', site.locales.map(&:to_s)].flatten(1) + end + + # @param :dir [String] + # @return [Array] + def files(dir) + Dir.chdir(dir) do + pages = Dir.glob('**/*.html') + files = Dir.glob('public/**/*') + files = remove_directories files + files = remove_images files + + [pages, files].flatten(1) + end + end + + # @param :files [Array] + # @return [Array] + def remove_directories(files) + files.reject do |f| + File.directory? f + end + end + + def remove_images(files) + files.reject do |f| + IMAGES.include? File.extname(f).downcase end end end From f94ece78cfe9c9cbf1a2ae65effed69b9000acff Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 19:06:17 -0300 Subject: [PATCH 083/164] optimizar sql usando indices --- app/jobs/uri_collection_job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index ada61bab..1bc6f768 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -37,6 +37,7 @@ class UriCollectionJob < PeriodicJob # intervalo más amplio. Stat::INTERVALS.reduce do |previous, current| Rollup.where(name: 'host|uri', interval: previous) + .where('time >= ?', beginning_of_interval) .where_dimensions(host: hostname, uri: uri) .group("dimensions->'host'", "dimensions->'uri'") .rollup('host|uri', interval: current, update: true) do |rollup| From 7cbc48d45b139c25656d8c0d8315d513d133181c Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 19:24:06 -0300 Subject: [PATCH 084/164] actualizar todo el intervalo --- app/jobs/uri_collection_job.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 1bc6f768..4d369c2d 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -36,8 +36,10 @@ class UriCollectionJob < PeriodicJob # Reducir las estadísticas calculadas aplicando un rollup sobre el # intervalo más amplio. Stat::INTERVALS.reduce do |previous, current| + beginning_of_this_interval = beginning_of_interval.try(:"beginning_of_#{current}") + Rollup.where(name: 'host|uri', interval: previous) - .where('time >= ?', beginning_of_interval) + .where('time >= ?', beginning_of_this_interval) .where_dimensions(host: hostname, uri: uri) .group("dimensions->'host'", "dimensions->'uri'") .rollup('host|uri', interval: current, update: true) do |rollup| From d8060728be781f074d9a8442408d651ef44d570c Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 19:32:55 -0300 Subject: [PATCH 085/164] =?UTF-8?q?mantener=20el=20d=C3=ADa=20anterior=20c?= =?UTF-8?q?ompleto=20en=20la=20base=20de=20datos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/access_logs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/access_logs b/bin/access_logs index 2b4fd99f..a7c151cc 100755 --- a/bin/access_logs +++ b/bin/access_logs @@ -1,8 +1,8 @@ #!/bin/sh set -e -# Volcar y eliminar todos los access logs del día anterior -date=`date +%F` +# Volcar y eliminar todos los access logs de dos días atrás +date=`bundle exec rails runner "puts (Date.today - 2.days)"` psql -h postgresql "${DATABASE:-sutty}" sutty < "/srv/http/_storage/${date}.psql.gz" begin; From 6b191d2d91541df6ef40e36a354e3a3a071b3ef5 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 19:37:08 -0300 Subject: [PATCH 086/164] =?UTF-8?q?usar=20la=20misma=20optimizaci=C3=B3n?= =?UTF-8?q?=20para=20stats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index e70544c3..3b68b61e 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -41,6 +41,7 @@ class StatCollectionJob < PeriodicJob # @return [NilClass] def rollup(name:, interval_previous:, interval:, operation: :sum) Rollup.where(name: name, interval: interval_previous) + .where('time >= ?', beginning_of_interval.try(:"beginning_of_#{interval}")) .where_dimensions(site_id: site.id) .group("dimensions->'site_id'") .rollup(name, interval: interval, update: true) do |rollup| From aad5c82cb7b7de9b673d2f4d57b41c3d39bef961 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 20:32:20 -0300 Subject: [PATCH 087/164] refactorizar rollups recursivos --- app/jobs/concerns/recursive_rollup.rb | 38 +++++++++++++ app/jobs/stat_collection_job.rb | 24 +++----- app/jobs/uri_collection_job.rb | 82 +++++++++++++++++++-------- 3 files changed, 102 insertions(+), 42 deletions(-) create mode 100644 app/jobs/concerns/recursive_rollup.rb diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb new file mode 100644 index 00000000..3163558e --- /dev/null +++ b/app/jobs/concerns/recursive_rollup.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module RecursiveRollup + extend ActiveSupport::Concern + + included do + private + + # Genera un rollup recursivo en base al período anterior y aplica una + # operación. + # + # @param :name [String] + # @param :interval_previous [String] + # @param :interval [String] + # @param :operation [Symbol] + # @param :dimensions [Hash] + # @param :new_name [String] + # @param :beginning [Time] + # @return [Rollup] + def recursive_rollup(name:, interval_previous:, interval:, dimensions:, beginning:, operation: :sum, new_name: nil) + Rollup.where(name: name, interval: interval_previous) + .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) + .where_dimensions(**dimensions) + .group(*dimensions_to_jsonb_query(dimensions)) + .rollup(new_name || name, interval: interval, update: true) do |rollup| + rollup.try(operation, :value) + end + end + + # @param :dimensions [Hash] + # @return [Array] + def dimensions_to_jsonb_query(dimensions) + dimensions.keys.map do |key| + "dimensions->'#{key}'" + end + end + end +end diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 3b68b61e..c94320d5 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -3,6 +3,8 @@ # Genera resúmenes de información para poder mostrar estadísticas y se # corre regularmente a sí misma. class StatCollectionJob < PeriodicJob + include RecursiveRollup + STAT_NAME = 'stat_collection_job' def perform(site_id:, once: true) @@ -20,9 +22,11 @@ class StatCollectionJob < PeriodicJob # XXX: Es correcto promediar promedios? Stat::INTERVALS.reduce do |previous, current| - rollup(name: 'builds', interval_previous: previous, interval: current) - rollup(name: 'space_used', interval_previous: previous, interval: current, operation: :average) - rollup(name: 'build_time', interval_previous: previous, interval: current, operation: :average) + opts = { interval_previous: previous, interval: current, dimensions: { site_id: site.id } } + + recursive_rollup(name: 'builds', **opts) + recursive_rollup(name: 'space_used', operation: :average, **opts) + recursive_rollup(name: 'build_time', operation: :average, **opts) current end @@ -35,20 +39,6 @@ class StatCollectionJob < PeriodicJob private - # Genera un rollup recursivo en base al período anterior y aplica una - # operación. - # - # @return [NilClass] - def rollup(name:, interval_previous:, interval:, operation: :sum) - Rollup.where(name: name, interval: interval_previous) - .where('time >= ?', beginning_of_interval.try(:"beginning_of_#{interval}")) - .where_dimensions(site_id: site.id) - .group("dimensions->'site_id'") - .rollup(name, interval: interval, update: true) do |rollup| - rollup.try(operation, :value) - end - end - # Los registros a procesar # # @return [ActiveRecord::Relation] diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 4d369c2d..cad05cbb 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -11,6 +11,8 @@ # Los hostnames de un sitio van a poder obtenerse a partir de # Site#hostnames con la garantía de que son únicos. class UriCollectionJob < PeriodicJob + include RecursiveRollup + # Ignoramos imágenes porque suelen ser demasiadas y no aportan a las # estadísticas. IMAGES = %w[.png .jpg .jpeg .gif .webp .jfif].freeze @@ -21,35 +23,19 @@ class UriCollectionJob < PeriodicJob # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME + name = 'host|uri' + beginning = beginning_of_interval + + hostnames.each do |host| + break if stop? - hostnames.each do |hostname| uris.each do |uri| - next if stop? + break if stop? - AccessLog.where(host: hostname, uri: uri) - .where('created_at >= ?', beginning_of_interval) - .completed_requests - .non_robots - .group(:host, :uri) - .rollup('host|uri', interval: starting_interval, update: true) - - # Reducir las estadísticas calculadas aplicando un rollup sobre el - # intervalo más amplio. - Stat::INTERVALS.reduce do |previous, current| - beginning_of_this_interval = beginning_of_interval.try(:"beginning_of_#{current}") - - Rollup.where(name: 'host|uri', interval: previous) - .where('time >= ?', beginning_of_this_interval) - .where_dimensions(host: hostname, uri: uri) - .group("dimensions->'host'", "dimensions->'uri'") - .rollup('host|uri', interval: current, update: true) do |rollup| - rollup.sum(:value) - end - - # Devolver el intervalo actual - current - end + rollup_uri(uri, host, name, beginning) end + + rollup_host(host, name, beginning) end stat.touch @@ -59,6 +45,52 @@ class UriCollectionJob < PeriodicJob private + def rollup_uri(uri, host, name, beginning) + dimensions = { host: host, uri: uri } + + AccessLog.where(**dimensions) + .where('created_at >= ?', beginning) + .completed_requests + .non_robots + .group(*dimensions.keys) + .rollup(name, interval: starting_interval, update: true) + + # Reducir las estadísticas calculadas aplicando un rollup sobre el + # intervalo más amplio. + Stat::INTERVALS.reduce do |previous, current| + recursive_rollup(name: name, + interval_previous: previous, + interval: current, + dimensions: dimensions, + beginning: beginning) + + # Devolver el intervalo actual + current + end + end + + def rollup_host(host, name, beginning) + dimensions = { host: host } + new_name = 'host' + + recursive_rollup(name: name, + new_name: new_name, + interval_previous: starting_interval, + interval: starting_interval, + dimensions: dimensions, + beginning: beginning) + + Stat::INTERVALS.reduce do |previous, current| + recursive_rollup(name: new_name, + interval_previous: previous, + interval: current, + dimensions: dimensions, + beginning: beginning) + + current + end + end + def stat_name STAT_NAME end From c09df3b87b8fa563e9225de4bed244ed4e280937 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 20:33:02 -0300 Subject: [PATCH 088/164] =?UTF-8?q?registrar=20el=20tiempo=20que=20pas?= =?UTF-8?q?=C3=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index c94320d5..03dcd292 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -10,6 +10,9 @@ class StatCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id + # Registrar que se hicieron todas las recolecciones + stat = site.stats.create! name: STAT_NAME + scope.rollup('builds', **options) scope.rollup('space_used', **options) do |rollup| @@ -31,9 +34,7 @@ class StatCollectionJob < PeriodicJob current end - # Registrar que se hicieron todas las recolecciones - site.stats.create! name: STAT_NAME - + stat.touch run_again! unless once end From 47a74b584824001e81d1bc1be86c6333ba2099b0 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 20:33:58 -0300 Subject: [PATCH 089/164] fixup! refactorizar rollups recursivos --- app/jobs/stat_collection_job.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 03dcd292..fcb4d6e1 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -12,6 +12,7 @@ class StatCollectionJob < PeriodicJob # Registrar que se hicieron todas las recolecciones stat = site.stats.create! name: STAT_NAME + beginning = beginning_of_interval scope.rollup('builds', **options) @@ -25,7 +26,7 @@ class StatCollectionJob < PeriodicJob # XXX: Es correcto promediar promedios? Stat::INTERVALS.reduce do |previous, current| - opts = { interval_previous: previous, interval: current, dimensions: { site_id: site.id } } + opts = { interval_previous: previous, interval: current, beginning: beginning, dimensions: { site_id: site.id } } recursive_rollup(name: 'builds', **opts) recursive_rollup(name: 'space_used', operation: :average, **opts) From 298fd5bed14fa31c04dcf8d9cb5785c8c86c3c36 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:46:48 -0300 Subject: [PATCH 090/164] centralizar la forma en que se obtienen los hostnames relacionado con !50 --- app/jobs/uri_collection_job.rb | 22 +--------------------- app/models/site.rb | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index cad05cbb..ea968fa0 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -26,7 +26,7 @@ class UriCollectionJob < PeriodicJob name = 'host|uri' beginning = beginning_of_interval - hostnames.each do |host| + site.hostnames.each do |host| break if stop? uris.each do |uri| @@ -102,26 +102,6 @@ class UriCollectionJob < PeriodicJob @destination ||= site.deploys.find_by(type: 'DeployLocal').destination end - # TODO: Cambiar al mergear origin-referer - # - # @return [Array] - def hostnames - @hostnames ||= site.deploys.map do |deploy| - case deploy - when DeployLocal - site.hostname - when DeployWww - deploy.fqdn - when DeployAlternativeDomain - deploy.hostname.dup.tap do |h| - h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}") - end - when DeployHiddenService - deploy.onion - end - end.compact - end - # Recolecta todas las URIs menos imágenes # # @return [Array] diff --git a/app/models/site.rb b/app/models/site.rb index 5b78d625..bcc9e60b 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -101,6 +101,26 @@ class Site < ApplicationRecord "https://#{hostname}#{slash ? '/' : ''}" end + # TODO: Cambiar al mergear origin-referer + # + # @return [Array] + def hostnames + @hostnames ||= deploys.map do |deploy| + case deploy + when DeployLocal + hostname + when DeployWww + deploy.fqdn + when DeployAlternativeDomain + deploy.hostname.dup.tap do |h| + h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}") + end + when DeployHiddenService + deploy.onion + end + end.compact + end + # Obtiene los dominios alternativos # # @return Array @@ -123,7 +143,9 @@ class Site < ApplicationRecord # # @return Array def urls(slash: true) - alternative_urls(slash: slash) << url(slash: slash) + @urls ||= hostnames.map do |h| + "https://#{h}#{slash ? '/' : ''}" + end end def invitade?(usuarie) From fb5cc0aaa7c10749d78ce18b2246c7b20945d273 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:48:25 -0300 Subject: [PATCH 091/164] breadcrumbs --- app/controllers/stats_controller.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 44073c1f..ba9113ff 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -8,6 +8,10 @@ class StatsController < ApplicationController before_action :authenticate_usuarie! before_action :authorize_stats + breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path + breadcrumb 'sites.index', :sites_path, match: :exact + breadcrumb -> { site.title }, -> { site_posts_path(site, locale: locale) }, match: :exact + EXTRA_OPTIONS = { builds: {}, space_used: { bytes: true }, @@ -21,6 +25,8 @@ class StatsController < ApplicationController end def index + breadcrumb I18n.t('stats.index.title'), '' + @chart_params = { interval: interval } hostnames last_stat From f72d27c3083ee32b50da0baa8e948566de65e79f Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:49:26 -0300 Subject: [PATCH 092/164] =?UTF-8?q?optimizaci=C3=B3n:=20usar=20las=20dimen?= =?UTF-8?q?siones=20como=20=C3=ADndices=20y=20no=20como=20texto?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Rollup#where_dimensions` no aprovecha los índices. --- app/jobs/concerns/recursive_rollup.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index 3163558e..b3327a73 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# Implementa rollups recursivos module RecursiveRollup extend ActiveSupport::Concern @@ -18,9 +19,8 @@ module RecursiveRollup # @param :beginning [Time] # @return [Rollup] def recursive_rollup(name:, interval_previous:, interval:, dimensions:, beginning:, operation: :sum, new_name: nil) - Rollup.where(name: name, interval: interval_previous) + Rollup.where(name: name, interval: interval_previous, dimensions: dimensions.to_json) .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) - .where_dimensions(**dimensions) .group(*dimensions_to_jsonb_query(dimensions)) .rollup(new_name || name, interval: interval, update: true) do |rollup| rollup.try(operation, :value) From 7c7ae2f75041b667a8f77ecbbf43fe2c569809d8 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:52:27 -0300 Subject: [PATCH 093/164] =?UTF-8?q?mostrar=20un=20per=C3=ADodo=20por=20def?= =?UTF-8?q?ecto?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit todavía no hay interfaz para cambiarlos --- app/controllers/stats_controller.rb | 30 +++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index ba9113ff..ded3a05c 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -24,6 +24,13 @@ class StatsController < ApplicationController policy.script_src :self, :unsafe_inline end + # Parámetros por defecto + # + # @return [Hash] + def default_url_options + { interval: 'day', period_start: Date.today.beginning_of_year, period_end: Date.today } + end + def index breadcrumb I18n.t('stats.index.title'), '' @@ -36,9 +43,9 @@ class StatsController < ApplicationController # Genera un gráfico de visitas por dominio asociado a este sitio def host - return unless stale? [last_stat, hostnames, interval] + return unless stale? [last_stat, hostnames, interval, period] - stats = Rollup.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series| + stats = rollup_scope.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series| series.each do |serie| serie[:name] = serie.dig(:dimensions, 'host') serie[:data].transform_values! do |value| @@ -51,7 +58,7 @@ class StatsController < ApplicationController end def resources - return unless stale? [last_stat, interval, resource] + return unless stale? [last_stat, interval, resource, period] options = { interval: interval, @@ -60,14 +67,14 @@ class StatsController < ApplicationController } } - render json: Rollup.series(resource, **options) + render json: rollup_scope.series(resource, **options) end def uris - return unless stale? [last_stat, hostnames, interval, normalized_urls] + return unless stale? [last_stat, hostnames, interval, normalized_urls, period] options = { host: hostnames, uri: normalized_paths } - stats = Rollup.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series| + stats = rollup_scope.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series| series.each do |serie| serie[:name] = serie[:dimensions].slice('host', 'uri').values.join.sub('/index.html', '/') serie[:data].transform_values! do |value| @@ -81,6 +88,10 @@ class StatsController < ApplicationController private + def rollup_scope + Rollup.where(time: period) + end + def last_stat @last_stat ||= Stat.last end @@ -171,4 +182,11 @@ class StatsController < ApplicationController def nodes @nodes ||= ENV.fetch('NODES', 1).to_i end + + def period + @period ||= begin + p = params.permit(:period_start, :period_end) + p[:period_start]..p[:period_end] + end + end end From 3a32422e26695feb58d4c45b5526d1bca227dfb8 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:54:04 -0300 Subject: [PATCH 094/164] =?UTF-8?q?usar=20m=C3=A9todos=20del=20sitio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index ded3a05c..116bca7c 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -103,23 +103,29 @@ class StatsController < ApplicationController # TODO: Eliminar cuando mergeemos referer-origin def hostnames - @hostnames ||= [@site.hostname, @site.alternative_hostnames].flatten + @hostnames ||= site.hostnames end # Normalizar las URLs # # @return [Array] def normalized_urls - @normalized_urls ||= params.permit(:urls).try(:[], - :urls)&.split("\n")&.map(&:strip)&.select(&:present?)&.select do |uri| - uri.start_with? 'https://' - end&.map do |u| - # XXX: Eliminar - # @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1} - next u unless u.end_with? '/' + @normalized_urls ||= + begin + urls = params.permit(:urls).try(:[], :urls)&.split("\n")&.map(&:strip)&.select(&:present?)&.select do |uri| + uri.start_with? 'https://' + end - "#{u}index.html" - end&.uniq || [@site.url, @site.urls].flatten.uniq + urls ||= [site.url] + + urls.map do |u| + # XXX: Eliminar al deployear + # @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1} + next u unless u.end_with? '/' + + "#{u}index.html" + end.uniq + end end def normalized_paths From 027d24d66a83dbd00273c3310e121090ade72097 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:55:54 -0300 Subject: [PATCH 095/164] memoizar el sitio --- app/controllers/stats_controller.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 116bca7c..10b75819 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -97,8 +97,7 @@ class StatsController < ApplicationController end def authorize_stats - @site = find_site - authorize SiteStat.new(@site) + authorize SiteStat.new(site) end # TODO: Eliminar cuando mergeemos referer-origin @@ -195,4 +194,8 @@ class StatsController < ApplicationController p[:period_start]..p[:period_end] end end + + def site + @site ||= find_site + end end From 0754e29a5a272a276ebefb93ab620838e9cc2fca Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:56:25 -0300 Subject: [PATCH 096/164] =?UTF-8?q?cada=20sitio=20tiene=20su=20propio=20co?= =?UTF-8?q?ntrol=20de=20estad=C3=ADsticas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 10b75819..4933c821 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -93,7 +93,7 @@ class StatsController < ApplicationController end def last_stat - @last_stat ||= Stat.last + @last_stat ||= site.stats.last end def authorize_stats From 7e575372cf1a7a42e4817f2ebe858941163daf6a Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:56:45 -0300 Subject: [PATCH 097/164] parametros por defecto --- app/controllers/stats_controller.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 4933c821..9a8a345d 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -162,14 +162,15 @@ class StatsController < ApplicationController def interval @interval ||= begin i = params[:interval]&.to_sym - Stat::INTERVALS.include?(i) ? i : :day + Stat::INTERVALS.include?(i) ? i : Stat::INTERVALS.first end end + # @return [Symbol] def resource @resource ||= begin r = params[:resource].to_sym - Stat::RESOURCES.include?(r) ? r : :builds + Stat::RESOURCES.include?(r) ? r : Stat::RESOURCES.first end end From a703eb409643fa6fa0d068d596472e7cfff5dac3 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:57:16 -0300 Subject: [PATCH 098/164] no fallar si el sitio no existe --- app/jobs/uri_collection_job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index ea968fa0..796a289c 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -20,6 +20,7 @@ class UriCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id + return unless File.directory? destination # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME From 5186892a87c0ec15aa1a4f9cb1dafa1b5173444c Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:57:57 -0300 Subject: [PATCH 099/164] =?UTF-8?q?fixup!=20usar=20m=C3=A9todos=20del=20si?= =?UTF-8?q?tio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/stats/index.haml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index bfcf33ef..2b5d643e 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -25,12 +25,12 @@ %input{ type: 'hidden', name: 'interval', value: @interval } .form-group %label{ for: 'urls' }= t('.urls.label') - %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size, aria_describedby: 'help-urls' }= @normalized_urls.join("\n") + %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size + 1, aria_describedby: 'help-urls' }= @normalized_urls.join("\n") %small#help-urls.feedback.form-text.text-muted= t('.urls.help') .form-group %button.btn{ type: 'submit' }= t('.urls.submit') - if @normalized_urls.present? - = line_chart site_stats_uris_path(urls: params[:urls], **@chart_params), **@chart_options + = line_chart site_stats_uris_path(urls: @normalized_urls, **@chart_params), **@chart_options .mb-5 %h2= t('.resources.title') From be6224ddf173f35d45a26df68abe71aea44282f2 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:59:02 -0300 Subject: [PATCH 100/164] =?UTF-8?q?recolectar=20estad=C3=ADsticas=20de=20r?= =?UTF-8?q?eferers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 46 +++++++++++++--------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 796a289c..12f25eb6 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -24,7 +24,6 @@ class UriCollectionJob < PeriodicJob # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME - name = 'host|uri' beginning = beginning_of_interval site.hostnames.each do |host| @@ -33,10 +32,18 @@ class UriCollectionJob < PeriodicJob uris.each do |uri| break if stop? - rollup_uri(uri, host, name, beginning) + rollup('host|uri', beginning, host: host, uri: uri) + + AccessLog.where(host: host, uri: uri).distinct(:http_referer).pluck(:http_referer).each do |http_referer| + rollup('host|uri|referer', beginning, host: host, uri: uri, http_referer: http_referer) + end end - rollup_host(host, name, beginning) + rollup('host', beginning, host: host) + + AccessLog.where(host: host).distinct(:http_referer).pluck(:http_referer).each do |http_referer| + rollup('host|referer', beginning, host: host, http_referer: http_referer) + end end stat.touch @@ -46,9 +53,14 @@ class UriCollectionJob < PeriodicJob private - def rollup_uri(uri, host, name, beginning) - dimensions = { host: host, uri: uri } - + # Generar un rollup a partir de unas dimensiones que también sirven de + # filtro. + # + # @param :name [String] + # @param :beginning [Time] + # @param :dimensions [Hash] + # @return [nil] + def rollup(name, beginning, **dimensions) AccessLog.where(**dimensions) .where('created_at >= ?', beginning) .completed_requests @@ -70,28 +82,6 @@ class UriCollectionJob < PeriodicJob end end - def rollup_host(host, name, beginning) - dimensions = { host: host } - new_name = 'host' - - recursive_rollup(name: name, - new_name: new_name, - interval_previous: starting_interval, - interval: starting_interval, - dimensions: dimensions, - beginning: beginning) - - Stat::INTERVALS.reduce do |previous, current| - recursive_rollup(name: new_name, - interval_previous: previous, - interval: current, - dimensions: dimensions, - beginning: beginning) - - current - end - end - def stat_name STAT_NAME end From 3b19d8f792f743964e7b4e2a2119fcb770bb8a97 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 18:50:00 -0300 Subject: [PATCH 101/164] =?UTF-8?q?actualizaci=C3=B3n=20diaria?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/locales/en.yml | 2 +- config/locales/es.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index 639d9184..732eb4bd 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -252,7 +252,7 @@ en: help: | These statistics show information about how your site is generated and how many resources it uses. - last_update: 'Updated every hour. Last update on ' + last_update: 'Updated daily. Last update on ' empty: 'There is no enough information yet. We invite you to come back in %{please_return_at}!' loading: 'Loading...' hour: 'Hourly' diff --git a/config/locales/es.yml b/config/locales/es.yml index 86e156df..e47c1fb5 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -257,7 +257,7 @@ es: help: | Las estadísticas visibilizan información sobre cómo se genera y cuántos recursos utiliza tu sitio. - last_update: 'Actualizadas cada hora. Última actualización hace ' + last_update: 'Actualizadas diariamente. Última actualización hace ' empty: 'Todavía no hay información suficiente. Te invitamos a volver en %{please_return_at} :)' loading: 'Cargando...' hour: 'Por hora' From 479ca7430f44269b643cd26ee6684f35544a9ceb Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 18:50:11 -0300 Subject: [PATCH 102/164] nombre --- app/jobs/stat_collection_job.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index fcb4d6e1..1ee2f2b3 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -57,4 +57,8 @@ class StatCollectionJob < PeriodicJob def options @options ||= { interval: starting_interval, update: true } end + + def stat_name + STAT_NAME + end end From 752935afb414d864c8acfe28184b0984c021b38f Mon Sep 17 00:00:00 2001 From: f Date: Wed, 27 Apr 2022 13:57:33 -0300 Subject: [PATCH 103/164] calcular bien la fecha --- Dockerfile | 2 +- app/jobs/stat_collection_job.rb | 2 +- bin/access_logs | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index ee6ba871..b3fd8e78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -114,7 +114,7 @@ RUN ln -s data/_private /srv/http/_private USER root # Instalar la configuración de monit RUN install -m 640 -o root -g root /srv/http/monit.conf /etc/monit.d/sutty.conf -RUN apk add --no-cache daemonize ruby-webrick +RUN apk add --no-cache daemonize ruby-webrick dateutils RUN install -m 755 /srv/http/entrypoint.sh /usr/local/bin/sutty # Mantener estos directorios! diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 1ee2f2b3..a67a3c50 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -47,7 +47,7 @@ class StatCollectionJob < PeriodicJob def scope @scope ||= site.build_stats .jekyll - .where('created_at => ?', beginning_of_interval) + .where('created_at >= ?', beginning_of_interval) .group(:site_id) end diff --git a/bin/access_logs b/bin/access_logs index a7c151cc..cfeeb57a 100755 --- a/bin/access_logs +++ b/bin/access_logs @@ -2,9 +2,12 @@ set -e # Volcar y eliminar todos los access logs de dos días atrás -date=`bundle exec rails runner "puts (Date.today - 2.days)"` +date="`dateadd today -1d`" +file="/srv/http/_storage/${date}.psql.gz" +test -n "${date}" +test ! -f "${file}" -psql -h postgresql "${DATABASE:-sutty}" sutty < "/srv/http/_storage/${date}.psql.gz" +psql -h postgresql "${DATABASE:-sutty}" sutty < "${file}" begin; copy (select * from access_logs where created_at < '${date}') to stdout; delete from access_logs where created_at < '${date}'; From eb7d0679bbc411d8f2afe831a23d958bc0e91abd Mon Sep 17 00:00:00 2001 From: f Date: Thu, 28 Apr 2022 10:34:57 -0300 Subject: [PATCH 104/164] normalizar strings sin romperlas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit en castellano no nos afectó pero en árabe rompe el significado --- .../active_storage/direct_uploads_controller_decorator.rb | 2 +- app/lib/action_dispatch/http/uploaded_file_decorator.rb | 2 +- app/models/metadata_markdown.rb | 2 +- app/models/metadata_markdown_content.rb | 2 +- app/models/metadata_permalink.rb | 2 +- app/models/metadata_string.rb | 2 +- app/models/metadata_template.rb | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/controllers/active_storage/direct_uploads_controller_decorator.rb b/app/controllers/active_storage/direct_uploads_controller_decorator.rb index 3052f974..c62dae2a 100644 --- a/app/controllers/active_storage/direct_uploads_controller_decorator.rb +++ b/app/controllers/active_storage/direct_uploads_controller_decorator.rb @@ -19,7 +19,7 @@ module ActiveStorage # stack. def blob_args params.require(:blob).permit(:filename, :byte_size, :checksum, :content_type, metadata: {}).to_h.symbolize_keys.tap do |ba| - ba[:filename] = ba[:filename].unicode_normalize(:nfkc) + ba[:filename] = ba[:filename].unicode_normalize end end end diff --git a/app/lib/action_dispatch/http/uploaded_file_decorator.rb b/app/lib/action_dispatch/http/uploaded_file_decorator.rb index c171c81c..0bdebdc0 100644 --- a/app/lib/action_dispatch/http/uploaded_file_decorator.rb +++ b/app/lib/action_dispatch/http/uploaded_file_decorator.rb @@ -11,7 +11,7 @@ module ActionDispatch # Devolver el nombre de archivo con caracteres unicode # normalizados def original_filename - @original_filename.unicode_normalize(:nfkc) + @original_filename.unicode_normalize end end end diff --git a/app/models/metadata_markdown.rb b/app/models/metadata_markdown.rb index 7816ec33..a09e351c 100644 --- a/app/models/metadata_markdown.rb +++ b/app/models/metadata_markdown.rb @@ -12,6 +12,6 @@ class MetadataMarkdown < MetadataText # markdown y se eliminan autolinks. Mejor es habilitar la generación # SAFE de CommonMark en la configuración del sitio. def sanitize(string) - string.unicode_normalize(:nfkc) + string.unicode_normalize end end diff --git a/app/models/metadata_markdown_content.rb b/app/models/metadata_markdown_content.rb index cb4124db..75088e30 100644 --- a/app/models/metadata_markdown_content.rb +++ b/app/models/metadata_markdown_content.rb @@ -25,6 +25,6 @@ class MetadataMarkdownContent < MetadataText # markdown y se eliminan autolinks. Mejor es deshabilitar la # generación SAFE de CommonMark en la configuración del sitio. def sanitize(string) - string.tr("\r", '').unicode_normalize(:nfkc) + string.tr("\r", '').unicode_normalize end end diff --git a/app/models/metadata_permalink.rb b/app/models/metadata_permalink.rb index 9b0c063c..30ad32cc 100644 --- a/app/models/metadata_permalink.rb +++ b/app/models/metadata_permalink.rb @@ -19,7 +19,7 @@ class MetadataPermalink < MetadataString # puntos suspensivos, la primera / para que siempre sea relativa y # agregamos una / al final si la ruta no tiene extensión. def sanitize(value) - value = value.strip.unicode_normalize(:nfkc).gsub('..', '/').gsub('./', '').squeeze('/') + value = value.strip.unicode_normalize.gsub('..', '/').gsub('./', '').squeeze('/') value = value[1..-1] if value.start_with? '/' value += '/' if File.extname(value).blank? diff --git a/app/models/metadata_string.rb b/app/models/metadata_string.rb index 28bfe82a..c1d888b1 100644 --- a/app/models/metadata_string.rb +++ b/app/models/metadata_string.rb @@ -17,7 +17,7 @@ class MetadataString < MetadataTemplate def sanitize(string) return '' if string.blank? - sanitizer.sanitize(string.strip.unicode_normalize(:nfkc), + sanitizer.sanitize(string.strip.unicode_normalize, tags: [], attributes: []).strip.html_safe end diff --git a/app/models/metadata_template.rb b/app/models/metadata_template.rb index a72f8e83..ddcd100e 100644 --- a/app/models/metadata_template.rb +++ b/app/models/metadata_template.rb @@ -185,7 +185,7 @@ MetadataTemplate = Struct.new(:site, :document, :name, :label, :type, return string unless string.is_a? String sanitizer - .sanitize(string.tr("\r", '').unicode_normalize(:nfkc), + .sanitize(string.tr("\r", '').unicode_normalize, tags: allowed_tags, attributes: allowed_attributes) .strip From f4ec30eac7c41d7ef189768cd929ac3928a7d184 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 28 Apr 2022 12:06:27 -0300 Subject: [PATCH 105/164] jekyll no soporta urls con barra al final MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit y genera un montón de problemas! --- app/models/site.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/site.rb b/app/models/site.rb index 5b78d625..af10b004 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -470,7 +470,7 @@ class Site < ApplicationRecord config.theme = design.gem unless design.no_theme? config.description = description config.title = title - config.url = url + config.url = url(slash: false) config.hostname = hostname end From 957c810aea703d7c877ca5521fca8a9c8f9f306f Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 15:04:42 -0300 Subject: [PATCH 106/164] =?UTF-8?q?recolectar=20informaci=C3=B3n=20acumul?= =?UTF-8?q?=C3=A1ndola?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 48 +++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 12f25eb6..2939b9f9 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -26,23 +26,43 @@ class UriCollectionJob < PeriodicJob stat = site.stats.create! name: STAT_NAME beginning = beginning_of_interval + # Recorremos todos los hostnames y uris posibles y luego agrupamos + # recursivamente para no tener que recalcular, asumiendo que es más + # rápido buscar en los rollups indexados que en la tabla en bruto. + # + # Los referers solo se agrupan por host. site.hostnames.each do |host| break if stop? + dimensions = { host: host } + uris.each do |uri| break if stop? - rollup('host|uri', beginning, host: host, uri: uri) + dimensions[:uri] = uri - AccessLog.where(host: host, uri: uri).distinct(:http_referer).pluck(:http_referer).each do |http_referer| - rollup('host|uri|referer', beginning, host: host, uri: uri, http_referer: http_referer) - end + rollup('host|uri', beginning, **dimensions) + reduce_rollup('host|uri', beginning, **dimensions) end - rollup('host', beginning, host: host) + dimensions.delete(:uri) + # Reducir todas las visitas a cantidad de visitas por host + recursive_rollup(name: 'host|uri', + new_name: 'host', + interval_previous: starting_interval, + interval: starting_interval, + dimensions: dimensions) + + # Acumular por mes y año + reduce_rollup('host', beginning, **dimensions) + + # Obtener orígenes de visitas por host AccessLog.where(host: host).distinct(:http_referer).pluck(:http_referer).each do |http_referer| - rollup('host|referer', beginning, host: host, http_referer: http_referer) + dimensions[:http_referer] = http_referer + + rollup('host|referer', beginning, **dimensions) + reduce_rollup('host|referer', beginning, **dimensions) end end @@ -53,8 +73,7 @@ class UriCollectionJob < PeriodicJob private - # Generar un rollup a partir de unas dimensiones que también sirven de - # filtro. + # Generar un rollup de access logs # # @param :name [String] # @param :beginning [Time] @@ -67,9 +86,16 @@ class UriCollectionJob < PeriodicJob .non_robots .group(*dimensions.keys) .rollup(name, interval: starting_interval, update: true) + end - # Reducir las estadísticas calculadas aplicando un rollup sobre el - # intervalo más amplio. + # Reducir las estadísticas calculadas aplicando un rollup sobre el + # intervalo más amplio. + # + # @param :name [String] + # @param :beginning [Time] + # @param :dimensions [Hash] + # @return [nil] + def reduce_rollup(name, beginning, **dimensions) Stat::INTERVALS.reduce do |previous, current| recursive_rollup(name: name, interval_previous: previous, @@ -80,6 +106,8 @@ class UriCollectionJob < PeriodicJob # Devolver el intervalo actual current end + + nil end def stat_name From 23260791e44cf747aa388d7cd1a496d9005639d7 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 15:29:27 -0300 Subject: [PATCH 107/164] =?UTF-8?q?fixup!=20recolectar=20informaci=C3=B3n?= =?UTF-8?q?=20acumul=C3=A1ndola?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 2939b9f9..bea388a7 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -52,7 +52,8 @@ class UriCollectionJob < PeriodicJob new_name: 'host', interval_previous: starting_interval, interval: starting_interval, - dimensions: dimensions) + dimensions: dimensions, + beginning: beginning) # Acumular por mes y año reduce_rollup('host', beginning, **dimensions) From cc839affb866b040e84e836a95be2294ab872934 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 17:43:06 -0300 Subject: [PATCH 108/164] tener en cuenta todos los directorios posibles por ejemplo DeployLocalizedDomain en la otra rama tiene otras rutas --- app/jobs/uri_collection_job.rb | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index bea388a7..8504f006 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -115,11 +115,17 @@ class UriCollectionJob < PeriodicJob STAT_NAME end + # Obtiene todas las ubicaciones de archivos + # # @return [String] # # TODO: Cambiar al mergear origin-referer - def destination - @destination ||= site.deploys.find_by(type: 'DeployLocal').destination + def destinations + @destinations ||= site.deploys.map(&:destination).select do |d| + File.directory?(d) + end.map do |d| + File.realpath(d) + end.uniq end # Recolecta todas las URIs menos imágenes @@ -127,14 +133,16 @@ class UriCollectionJob < PeriodicJob # @return [Array] def uris @uris ||= - locales.map do |locale| - uri = "/#{locale}/".squeeze('/') - dir = File.join(destination, locale) + destinations.map do |destination| + locales.map do |locale| + uri = "/#{locale}/".squeeze('/') + dir = File.join(destination, locale) - files(dir).map do |f| - uri + f + files(dir).map do |f| + uri + f + end end - end.flatten(2) + end.flatten(3) end # @return [Array] From 9998f19087318910b70ad69e998bc1c0aaf809a2 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 17:57:29 -0300 Subject: [PATCH 109/164] =?UTF-8?q?acumular=20visitas=20por=20pa=C3=ADs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 8504f006..eaa3b153 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -58,12 +58,20 @@ class UriCollectionJob < PeriodicJob # Acumular por mes y año reduce_rollup('host', beginning, **dimensions) - # Obtener orígenes de visitas por host - AccessLog.where(host: host).distinct(:http_referer).pluck(:http_referer).each do |http_referer| - dimensions[:http_referer] = http_referer + columns = %i[http_referer geoip2_data_country_name] + columns.each do |column| + # Obtener orígenes de visitas por host + AccessLog.where(host: host).distinct(column).pluck(column).each do |value| + dimensions.delete_if do |k, _| + columns.include? k + end - rollup('host|referer', beginning, **dimensions) - reduce_rollup('host|referer', beginning, **dimensions) + name = "host|#{column}" + dimensions[column] = value + + rollup(name, beginning, **dimensions) + reduce_rollup(name, beginning, **dimensions) + end end end From 23e2edf259759ab0b85bddbb57bfa81bc2c0b677 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 18:01:07 -0300 Subject: [PATCH 110/164] =?UTF-8?q?la=20validaci=C3=B3n=20de=20directorio?= =?UTF-8?q?=20pasa=20por=20otro=20lado?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index eaa3b153..0c5b6b1f 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -20,7 +20,6 @@ class UriCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id - return unless File.directory? destination # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME From d56eeb465a8c91f8874a60d5babfe7baad8071ef Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 18:04:43 -0300 Subject: [PATCH 111/164] =?UTF-8?q?comprobar=20que=20los=20idiomas=20tambi?= =?UTF-8?q?=C3=A9n=20existan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 0c5b6b1f..b6d4a21b 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -145,11 +145,13 @@ class UriCollectionJob < PeriodicJob uri = "/#{locale}/".squeeze('/') dir = File.join(destination, locale) + next unless File.directory? dir + files(dir).map do |f| uri + f end end - end.flatten(3) + end.flatten(3).compact end # @return [Array] From 7dad3e9e54f523f7d4b1e801aede6dd4bfb40e5e Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 20:06:07 -0300 Subject: [PATCH 112/164] =?UTF-8?q?ser=20m=C3=A1s=20explicitxs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index b6d4a21b..d8b24e3a 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -33,39 +33,30 @@ class UriCollectionJob < PeriodicJob site.hostnames.each do |host| break if stop? - dimensions = { host: host } - uris.each do |uri| break if stop? - dimensions[:uri] = uri - - rollup('host|uri', beginning, **dimensions) - reduce_rollup('host|uri', beginning, **dimensions) + rollup('host|uri', beginning, host: host, uri: uri) + reduce_rollup('host|uri', beginning, host: host, uri: uri) end - dimensions.delete(:uri) - # Reducir todas las visitas a cantidad de visitas por host recursive_rollup(name: 'host|uri', new_name: 'host', interval_previous: starting_interval, interval: starting_interval, - dimensions: dimensions, + dimensions: { host: host }, beginning: beginning) # Acumular por mes y año - reduce_rollup('host', beginning, **dimensions) + reduce_rollup('host', beginning, host: host) columns = %i[http_referer geoip2_data_country_name] columns.each do |column| # Obtener orígenes de visitas por host AccessLog.where(host: host).distinct(column).pluck(column).each do |value| - dimensions.delete_if do |k, _| - columns.include? k - end - name = "host|#{column}" + dimensions = { host: host } dimensions[column] = value rollup(name, beginning, **dimensions) From 595a8759a4379ea2e7bad904afe3bb46142207a7 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 20:32:21 -0300 Subject: [PATCH 113/164] usar las uris como fuente de verdad --- app/jobs/uri_collection_job.rb | 45 ++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index d8b24e3a..310984a0 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -24,6 +24,7 @@ class UriCollectionJob < PeriodicJob # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME beginning = beginning_of_interval + columns = %i[http_referer geoip2_data_country_name] # Recorremos todos los hostnames y uris posibles y luego agrupamos # recursivamente para no tener que recalcular, asumiendo que es más @@ -33,11 +34,31 @@ class UriCollectionJob < PeriodicJob site.hostnames.each do |host| break if stop? + host_dimensions = { host: host } + + # Las URIs son la fuente de verdad de las visitas, porque son las + # que indican las páginas y recursos descargables, el resto son + # imágenes, CSS, JS y tipografías que no nos aportan números + # significativos. uris.each do |uri| break if stop? - rollup('host|uri', beginning, host: host, uri: uri) - reduce_rollup('host|uri', beginning, host: host, uri: uri) + name = 'host|uri' + dimensions = { host: host, uri: uri } + + rollup(name, beginning, **dimensions) + reduce_rollup(name, beginning, **dimensions) + + columns.each do |column| + # Obtener orígenes de visitas por host + AccessLog.where(**host_dimensions).distinct(column).pluck(column).each do |value| + name = "#{name}|#{column}" + dimensions[column] = value + + rollup(name, beginning, **dimensions) + reduce_rollup(name, beginning, **dimensions) + end + end end # Reducir todas las visitas a cantidad de visitas por host @@ -45,23 +66,21 @@ class UriCollectionJob < PeriodicJob new_name: 'host', interval_previous: starting_interval, interval: starting_interval, - dimensions: { host: host }, + dimensions: host_dimensions, beginning: beginning) # Acumular por mes y año - reduce_rollup('host', beginning, host: host) + reduce_rollup('host', beginning, **host_dimensions) - columns = %i[http_referer geoip2_data_country_name] columns.each do |column| - # Obtener orígenes de visitas por host - AccessLog.where(host: host).distinct(column).pluck(column).each do |value| - name = "host|#{column}" - dimensions = { host: host } - dimensions[column] = value + recursive_rollup(name: "host|uri|#{column}", + new_name: "host|#{column}", + interval_previous: starting_interval, + interval: starting_interval, + dimensions: host_dimensions, + beginning: beginning) - rollup(name, beginning, **dimensions) - reduce_rollup(name, beginning, **dimensions) - end + reduce_rollup("host|#{column}", beginning, **host_dimensions) end end From 121d4f49e26505f2f4edb2ef095a6ead487d83ab Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 09:56:19 -0300 Subject: [PATCH 114/164] no se puede renombrar un rollup usando dimensiones exactas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit además era un método que hacía dos cosas distintas según un parámetro --- app/jobs/concerns/recursive_rollup.rb | 27 +++++++++++++++++++++++---- app/jobs/uri_collection_job.rb | 22 ++++++++++------------ 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index b3327a73..5ba9dd8b 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -15,14 +15,33 @@ module RecursiveRollup # @param :interval [String] # @param :operation [Symbol] # @param :dimensions [Hash] - # @param :new_name [String] # @param :beginning [Time] # @return [Rollup] - def recursive_rollup(name:, interval_previous:, interval:, dimensions:, beginning:, operation: :sum, new_name: nil) - Rollup.where(name: name, interval: interval_previous, dimensions: dimensions.to_json) + def recursive_rollup(name:, interval_previous:, interval:, dimensions:, beginning:, operation: :sum) + Rollup.where(name: name, interval: interval_previous, dimensions: dimensions) .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) .group(*dimensions_to_jsonb_query(dimensions)) - .rollup(new_name || name, interval: interval, update: true) do |rollup| + .rollup(name, interval: interval, update: true) do |rollup| + rollup.try(operation, :value) + end + end + + # Genera un nuevo rollup a partir de uno anterior. + # + # @param :name [String] + # @param :new_name [String] + # @param :interval_previous [String] + # @param :interval [String] + # @param :operation [Symbol] + # @param :dimensions [Hash] + # @param :beginning [Time] + # @return [Rollup] + def square_rollup(name:, new_name:, interval:, dimensions:, beginning:, operation: :sum) + Rollup.where(name: name, interval: interval_previous) + .where_dimensions(**dimensions) + .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) + .group(*dimensions_to_jsonb_query(dimensions)) + .rollup(new_name, interval: interval, update: true) do |rollup| rollup.try(operation, :value) end end diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 310984a0..5de95d43 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -62,23 +62,21 @@ class UriCollectionJob < PeriodicJob end # Reducir todas las visitas a cantidad de visitas por host - recursive_rollup(name: 'host|uri', - new_name: 'host', - interval_previous: starting_interval, - interval: starting_interval, - dimensions: host_dimensions, - beginning: beginning) + square_rollup(name: 'host|uri', + new_name: 'host', + interval: starting_interval, + dimensions: host_dimensions, + beginning: beginning) # Acumular por mes y año reduce_rollup('host', beginning, **host_dimensions) columns.each do |column| - recursive_rollup(name: "host|uri|#{column}", - new_name: "host|#{column}", - interval_previous: starting_interval, - interval: starting_interval, - dimensions: host_dimensions, - beginning: beginning) + square_rollup(name: "host|uri|#{column}", + new_name: "host|#{column}", + interval: starting_interval, + dimensions: host_dimensions, + beginning: beginning) reduce_rollup("host|#{column}", beginning, **host_dimensions) end From b36bdc976324d7359c8ac2a0574ff6e949fdb559 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 10:03:14 -0300 Subject: [PATCH 115/164] fixup! no se puede renombrar un rollup usando dimensiones exactas --- app/jobs/concerns/recursive_rollup.rb | 6 +++--- app/jobs/uri_collection_job.rb | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index 5ba9dd8b..3539d7d2 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -30,14 +30,14 @@ module RecursiveRollup # # @param :name [String] # @param :new_name [String] - # @param :interval_previous [String] - # @param :interval [String] + # @param :interval_previous [String, Symbol] + # @param :interval [String, Symbol] # @param :operation [Symbol] # @param :dimensions [Hash] # @param :beginning [Time] # @return [Rollup] def square_rollup(name:, new_name:, interval:, dimensions:, beginning:, operation: :sum) - Rollup.where(name: name, interval: interval_previous) + Rollup.where(name: name, interval: interval) .where_dimensions(**dimensions) .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) .group(*dimensions_to_jsonb_query(dimensions)) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 5de95d43..a9549efe 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -52,11 +52,11 @@ class UriCollectionJob < PeriodicJob columns.each do |column| # Obtener orígenes de visitas por host AccessLog.where(**host_dimensions).distinct(column).pluck(column).each do |value| - name = "#{name}|#{column}" + column_name = "host|#{column}" dimensions[column] = value - rollup(name, beginning, **dimensions) - reduce_rollup(name, beginning, **dimensions) + rollup(column_name, beginning, **dimensions) + reduce_rollup(column_name, beginning, **dimensions) end end end @@ -145,6 +145,9 @@ class UriCollectionJob < PeriodicJob # Recolecta todas las URIs menos imágenes # + # TODO: Para los sitios con DeployLocalizedDomain estamos buscando + # URIs de más. + # # @return [Array] def uris @uris ||= From ce0d2306f6f8b14ea046c77d41ff9a106559bae4 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 10:13:49 -0300 Subject: [PATCH 116/164] fixup! fixup! no se puede renombrar un rollup usando dimensiones exactas --- app/jobs/uri_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index a9549efe..1f635997 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -52,7 +52,7 @@ class UriCollectionJob < PeriodicJob columns.each do |column| # Obtener orígenes de visitas por host AccessLog.where(**host_dimensions).distinct(column).pluck(column).each do |value| - column_name = "host|#{column}" + column_name = "host|uri|#{column}" dimensions[column] = value rollup(column_name, beginning, **dimensions) From deb3f0f0709bed4d586e8580cb1987722b310094 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 10:23:58 -0300 Subject: [PATCH 117/164] =?UTF-8?q?usar=20el=20=C3=BAltimo=20intervalo,=20?= =?UTF-8?q?no=20el=20que=20acabamos=20de=20crear?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 1f635997..e5105e5a 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -21,9 +21,10 @@ class UriCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id + # Obtener el principio del intervalo anterior + beginning = beginning_of_interval # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME - beginning = beginning_of_interval columns = %i[http_referer geoip2_data_country_name] # Recorremos todos los hostnames y uris posibles y luego agrupamos From 80f2df362b3091b8c9856766ceb1bf4bd5dc4f64 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 10:24:21 -0300 Subject: [PATCH 118/164] precalcular las columnas --- app/jobs/uri_collection_job.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index e5105e5a..196d2e50 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -25,7 +25,8 @@ class UriCollectionJob < PeriodicJob beginning = beginning_of_interval # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME - columns = %i[http_referer geoip2_data_country_name] + # Columnas a agrupar + columns = %i[http_referer geoip2_data_country_name].zip([nil]).to_h # Recorremos todos los hostnames y uris posibles y luego agrupamos # recursivamente para no tener que recalcular, asumiendo que es más @@ -36,6 +37,9 @@ class UriCollectionJob < PeriodicJob break if stop? host_dimensions = { host: host } + columns.each_key do |column| + columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) + end # Las URIs son la fuente de verdad de las visitas, porque son las # que indican las páginas y recursos descargables, el resto son @@ -50,9 +54,9 @@ class UriCollectionJob < PeriodicJob rollup(name, beginning, **dimensions) reduce_rollup(name, beginning, **dimensions) - columns.each do |column| + columns.each_pair do |column, values| # Obtener orígenes de visitas por host - AccessLog.where(**host_dimensions).distinct(column).pluck(column).each do |value| + values.each do |value| column_name = "host|uri|#{column}" dimensions[column] = value @@ -72,7 +76,7 @@ class UriCollectionJob < PeriodicJob # Acumular por mes y año reduce_rollup('host', beginning, **host_dimensions) - columns.each do |column| + columns.each_key do |column| square_rollup(name: "host|uri|#{column}", new_name: "host|#{column}", interval: starting_interval, From e9d13a7c5931b9ad4e86bf218ff6aff79ac132c0 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 10:49:32 -0300 Subject: [PATCH 119/164] mostrar demora --- app/jobs/uri_collection_job.rb | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 196d2e50..ae4b098e 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -38,7 +38,10 @@ class UriCollectionJob < PeriodicJob host_dimensions = { host: host } columns.each_key do |column| + time_start = Time.now columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) + + puts "#{column}: #{Time.now - time_start}" end # Las URIs son la fuente de verdad de las visitas, porque son las @@ -51,39 +54,58 @@ class UriCollectionJob < PeriodicJob name = 'host|uri' dimensions = { host: host, uri: uri } + time_start = Time.now rollup(name, beginning, **dimensions) + puts "#{dimensions.values.join}: #{Time.now - time_start}" + + time_start = Time.now reduce_rollup(name, beginning, **dimensions) + puts "reduced: #{Time.now - time_start}" columns.each_pair do |column, values| # Obtener orígenes de visitas por host values.each do |value| + time_start = Time.now + column_name = "host|uri|#{column}" dimensions[column] = value rollup(column_name, beginning, **dimensions) + puts "#{dimensions.values.join}: #{Time.now - time_start}" + + time_start = Time.now reduce_rollup(column_name, beginning, **dimensions) + puts "reduced: #{Time.now - time_start}" end end end # Reducir todas las visitas a cantidad de visitas por host + time_start = Time.now square_rollup(name: 'host|uri', new_name: 'host', interval: starting_interval, dimensions: host_dimensions, beginning: beginning) + puts "#{host}: #{Time.now - time_start}" # Acumular por mes y año + time_start = Time.now reduce_rollup('host', beginning, **host_dimensions) + puts "reduced: #{Time.now - time_start}" columns.each_key do |column| + time_start = Time.now square_rollup(name: "host|uri|#{column}", new_name: "host|#{column}", interval: starting_interval, dimensions: host_dimensions, beginning: beginning) + puts "#{host} #{column}: #{Time.now - time_start}" + time_start = Time.now reduce_rollup("host|#{column}", beginning, **host_dimensions) + puts "reduced: #{Time.now - time_start}" end end From 1962175cc66502848aa0e17116959c3e0986f140 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 10:58:41 -0300 Subject: [PATCH 120/164] Revert "mostrar demora" This reverts commit e9d13a7c5931b9ad4e86bf218ff6aff79ac132c0. --- app/jobs/uri_collection_job.rb | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index ae4b098e..196d2e50 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -38,10 +38,7 @@ class UriCollectionJob < PeriodicJob host_dimensions = { host: host } columns.each_key do |column| - time_start = Time.now columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) - - puts "#{column}: #{Time.now - time_start}" end # Las URIs son la fuente de verdad de las visitas, porque son las @@ -54,58 +51,39 @@ class UriCollectionJob < PeriodicJob name = 'host|uri' dimensions = { host: host, uri: uri } - time_start = Time.now rollup(name, beginning, **dimensions) - puts "#{dimensions.values.join}: #{Time.now - time_start}" - - time_start = Time.now reduce_rollup(name, beginning, **dimensions) - puts "reduced: #{Time.now - time_start}" columns.each_pair do |column, values| # Obtener orígenes de visitas por host values.each do |value| - time_start = Time.now - column_name = "host|uri|#{column}" dimensions[column] = value rollup(column_name, beginning, **dimensions) - puts "#{dimensions.values.join}: #{Time.now - time_start}" - - time_start = Time.now reduce_rollup(column_name, beginning, **dimensions) - puts "reduced: #{Time.now - time_start}" end end end # Reducir todas las visitas a cantidad de visitas por host - time_start = Time.now square_rollup(name: 'host|uri', new_name: 'host', interval: starting_interval, dimensions: host_dimensions, beginning: beginning) - puts "#{host}: #{Time.now - time_start}" # Acumular por mes y año - time_start = Time.now reduce_rollup('host', beginning, **host_dimensions) - puts "reduced: #{Time.now - time_start}" columns.each_key do |column| - time_start = Time.now square_rollup(name: "host|uri|#{column}", new_name: "host|#{column}", interval: starting_interval, dimensions: host_dimensions, beginning: beginning) - puts "#{host} #{column}: #{Time.now - time_start}" - time_start = Time.now reduce_rollup("host|#{column}", beginning, **host_dimensions) - puts "reduced: #{Time.now - time_start}" end end From 6699db838078d141cc3eaab7815b09d22d1862ec Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 11:34:21 -0300 Subject: [PATCH 121/164] otra vez el mismo error de hashes acumulados hice benchmarks, generar hashes nuevos es mas rapido que duplicarlos, y eliminar las keys que no corresponden es apenas mas rapido pero con mas posibilidad de bugs --- app/jobs/uri_collection_job.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 196d2e50..620123db 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -58,10 +58,11 @@ class UriCollectionJob < PeriodicJob # Obtener orígenes de visitas por host values.each do |value| column_name = "host|uri|#{column}" - dimensions[column] = value + column_dimensions = { host: host, uri: uri } + column_dimensions[column] = value - rollup(column_name, beginning, **dimensions) - reduce_rollup(column_name, beginning, **dimensions) + rollup(column_name, beginning, **column_dimensions) + reduce_rollup(column_name, beginning, **column_dimensions) end end end From 6bbb5b48c0c6f63fa3ee2601affcdd51d94cb79e Mon Sep 17 00:00:00 2001 From: Nulo Date: Sat, 30 Apr 2022 14:56:04 +0000 Subject: [PATCH 122/164] editor: Borrar comentario viejo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ahora la blockquote está implementada por https://0xacab.org/sutty/sutty/-/merge_requests/83, está implementada como bloque normal ya que una blockquote puede tener varios blockquotes adentro --- app/javascript/editor/types/parentBlocks.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/app/javascript/editor/types/parentBlocks.ts b/app/javascript/editor/types/parentBlocks.ts index ffe40bdf..5ba8f366 100644 --- a/app/javascript/editor/types/parentBlocks.ts +++ b/app/javascript/editor/types/parentBlocks.ts @@ -20,7 +20,6 @@ function makeParentBlock( }; } -// TODO: añadir blockquote // XXX: si agregás algo acá, probablemente le quieras hacer un botón // en app/views/posts/attributes/_content.haml export const parentBlocks: { [propName: string]: EditorNode } = { From 001443c1be55fc0f7f893392fd9e119a1c07c125 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 14:07:34 -0300 Subject: [PATCH 123/164] =?UTF-8?q?la=20=C3=BAltima=20actualizaci=C3=B3n?= =?UTF-8?q?=20es=20cuando=20se=20cierra=20el=20stat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/stats/index.haml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 2b5d643e..02683781 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -6,8 +6,8 @@ %p %small = t('.last_update') - %time{ datetime: @last_stat.created_at } - #{time_ago_in_words @last_stat.created_at}. + %time{ datetime: @last_stat.updated_at } + #{time_ago_in_words @last_stat.updated_at}. .mb-5 - Stat::INTERVALS.each do |interval| From 6bf286077bba19858665314f9fd850133229c7eb Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 14:39:31 -0300 Subject: [PATCH 124/164] =?UTF-8?q?generar=20estad=C3=ADsticas=20de=20uso?= =?UTF-8?q?=20de=20infra?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/concerns/recursive_rollup.rb | 24 +++++++++++++++++++++ app/jobs/stat_collection_job.rb | 22 +++++--------------- app/jobs/uri_collection_job.rb | 30 ++++----------------------- 3 files changed, 33 insertions(+), 43 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index 3539d7d2..28c2e66a 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -46,6 +46,30 @@ module RecursiveRollup end end + # Reducir las estadísticas calculadas aplicando un rollup sobre el + # intervalo más amplio. + # + # @param :name [String] + # @param :beginning [Time] + # @param :operation [Symbol] + # @param :dimensions [Hash] + # @return [nil] + def reduce_rollup(name, beginning, operation, **dimensions) + Stat::INTERVALS.reduce do |previous, current| + recursive_rollup(name: name, + interval_previous: previous, + interval: current, + dimensions: dimensions, + beginning: beginning, + operation: operation) + + # Devolver el intervalo actual + current + end + + nil + end + # @param :dimensions [Hash] # @return [Array] def dimensions_to_jsonb_query(dimensions) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index a67a3c50..65226b5c 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -9,10 +9,8 @@ class StatCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id - - # Registrar que se hicieron todas las recolecciones - stat = site.stats.create! name: STAT_NAME beginning = beginning_of_interval + stat = site.stats.create! name: STAT_NAME scope.rollup('builds', **options) @@ -24,16 +22,9 @@ class StatCollectionJob < PeriodicJob rollup.average(:seconds) end - # XXX: Es correcto promediar promedios? - Stat::INTERVALS.reduce do |previous, current| - opts = { interval_previous: previous, interval: current, beginning: beginning, dimensions: { site_id: site.id } } - - recursive_rollup(name: 'builds', **opts) - recursive_rollup(name: 'space_used', operation: :average, **opts) - recursive_rollup(name: 'build_time', operation: :average, **opts) - - current - end + reduce_rollup('builds', beginning, :sum, site_id: site_id) + reduce_rollup('space_used', beginning, :average, site_id: site_id) + reduce_rollup('build_time', beginning, :average, site_id: site_id) stat.touch run_again! unless once @@ -45,10 +36,7 @@ class StatCollectionJob < PeriodicJob # # @return [ActiveRecord::Relation] def scope - @scope ||= site.build_stats - .jekyll - .where('created_at >= ?', beginning_of_interval) - .group(:site_id) + @scope ||= site.build_stats.jekyll.where('created_at >= ?', beginning_of_interval) end # Las opciones por defecto diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 620123db..1553fb22 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -52,7 +52,7 @@ class UriCollectionJob < PeriodicJob dimensions = { host: host, uri: uri } rollup(name, beginning, **dimensions) - reduce_rollup(name, beginning, **dimensions) + reduce_rollup(name, beginning, :sum, **dimensions) columns.each_pair do |column, values| # Obtener orígenes de visitas por host @@ -62,7 +62,7 @@ class UriCollectionJob < PeriodicJob column_dimensions[column] = value rollup(column_name, beginning, **column_dimensions) - reduce_rollup(column_name, beginning, **column_dimensions) + reduce_rollup(column_name, beginning, :sum, **column_dimensions) end end end @@ -75,7 +75,7 @@ class UriCollectionJob < PeriodicJob beginning: beginning) # Acumular por mes y año - reduce_rollup('host', beginning, **host_dimensions) + reduce_rollup('host', beginning, :sum, **host_dimensions) columns.each_key do |column| square_rollup(name: "host|uri|#{column}", @@ -84,7 +84,7 @@ class UriCollectionJob < PeriodicJob dimensions: host_dimensions, beginning: beginning) - reduce_rollup("host|#{column}", beginning, **host_dimensions) + reduce_rollup("host|#{column}", beginning, :sum, **host_dimensions) end end @@ -110,28 +110,6 @@ class UriCollectionJob < PeriodicJob .rollup(name, interval: starting_interval, update: true) end - # Reducir las estadísticas calculadas aplicando un rollup sobre el - # intervalo más amplio. - # - # @param :name [String] - # @param :beginning [Time] - # @param :dimensions [Hash] - # @return [nil] - def reduce_rollup(name, beginning, **dimensions) - Stat::INTERVALS.reduce do |previous, current| - recursive_rollup(name: name, - interval_previous: previous, - interval: current, - dimensions: dimensions, - beginning: beginning) - - # Devolver el intervalo actual - current - end - - nil - end - def stat_name STAT_NAME end From 8d4958af536a6e9a0dedcd9381811e5898fee91f Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 14:44:27 -0300 Subject: [PATCH 125/164] =?UTF-8?q?fixup!=20generar=20estad=C3=ADsticas=20?= =?UTF-8?q?de=20uso=20de=20infra?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 65226b5c..3f7cdd48 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -36,7 +36,7 @@ class StatCollectionJob < PeriodicJob # # @return [ActiveRecord::Relation] def scope - @scope ||= site.build_stats.jekyll.where('created_at >= ?', beginning_of_interval) + @scope ||= site.build_stats.jekyll.where('created_at >= ?', beginning_of_interval).group(:site_id) end # Las opciones por defecto From 92cd2ac05236c206d69cf081bec8d9e3fd658dc2 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 14:48:42 -0300 Subject: [PATCH 126/164] =?UTF-8?q?fixup!=20fixup!=20generar=20estad=C3=AD?= =?UTF-8?q?sticas=20de=20uso=20de=20infra?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 3f7cdd48..63d44999 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -36,7 +36,7 @@ class StatCollectionJob < PeriodicJob # # @return [ActiveRecord::Relation] def scope - @scope ||= site.build_stats.jekyll.where('created_at >= ?', beginning_of_interval).group(:site_id) + @scope ||= site.build_stats.jekyll.where('build_stats.created_at >= ?', beginning_of_interval).group(:site_id) end # Las opciones por defecto From 756fb8f1cbc32561bfd1433b0698267024d870ff Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 16:38:16 -0300 Subject: [PATCH 127/164] =?UTF-8?q?usar=20el=20id=20del=20sitio=20como=20d?= =?UTF-8?q?imensi=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 9a8a345d..3e1ec76b 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -60,12 +60,7 @@ class StatsController < ApplicationController def resources return unless stale? [last_stat, interval, resource, period] - options = { - interval: interval, - dimensions: { - deploy_id: @site.deploys.where(type: 'DeployLocal').pluck(:id).first - } - } + options = { interval: interval, dimensions: { site_id: site.id } } render json: rollup_scope.series(resource, **options) end From d181c179ff48948143526f4e63c6ac8b9bc40d75 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 16:38:32 -0300 Subject: [PATCH 128/164] nota --- app/controllers/stats_controller.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 3e1ec76b..35f7f108 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -69,6 +69,8 @@ class StatsController < ApplicationController return unless stale? [last_stat, hostnames, interval, normalized_urls, period] options = { host: hostnames, uri: normalized_paths } + # XXX: where_dimensions es más corto pero no aprovecha los índices + # de Rollup stats = rollup_scope.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series| series.each do |serie| serie[:name] = serie[:dimensions].slice('host', 'uri').values.join.sub('/index.html', '/') From 604ee233bd1ef7bbbfa5ccd293a7d2edd6ca248e Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 16:38:50 -0300 Subject: [PATCH 129/164] =?UTF-8?q?agregar=20la=20dimensi=C3=B3n=20al=20ag?= =?UTF-8?q?rupar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 1553fb22..44642c1f 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -78,13 +78,18 @@ class UriCollectionJob < PeriodicJob reduce_rollup('host', beginning, :sum, **host_dimensions) columns.each_key do |column| - square_rollup(name: "host|uri|#{column}", - new_name: "host|#{column}", - interval: starting_interval, - dimensions: host_dimensions, - beginning: beginning) + values.each do |value| + column_dimensions = { host: host } + column_dimensions[column] = value - reduce_rollup("host|#{column}", beginning, :sum, **host_dimensions) + square_rollup(name: "host|uri|#{column}", + new_name: "host|#{column}", + interval: starting_interval, + dimensions: column_dimensions, + beginning: beginning) + + reduce_rollup("host|#{column}", beginning, :sum, **column_dimensions) + end end end From d766ee4eff68284e28462f1afe3e9c6c6d061698 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 16:44:23 -0300 Subject: [PATCH 130/164] =?UTF-8?q?fixup!=20agregar=20la=20dimensi=C3=B3n?= =?UTF-8?q?=20al=20agrupar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 44642c1f..a0b1181e 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -77,7 +77,7 @@ class UriCollectionJob < PeriodicJob # Acumular por mes y año reduce_rollup('host', beginning, :sum, **host_dimensions) - columns.each_key do |column| + columns.each_pair do |column, values| values.each do |value| column_dimensions = { host: host } column_dimensions[column] = value From fb34823ce31606c23823d66365ef85438e0e12fa Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 17:51:05 -0300 Subject: [PATCH 131/164] deprecar square_rollup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit es muy lento buscar dentro de cada dimensión --- app/jobs/concerns/recursive_rollup.rb | 20 -------------------- app/jobs/uri_collection_job.rb | 13 ++----------- 2 files changed, 2 insertions(+), 31 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index 28c2e66a..4d9e11b0 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -26,26 +26,6 @@ module RecursiveRollup end end - # Genera un nuevo rollup a partir de uno anterior. - # - # @param :name [String] - # @param :new_name [String] - # @param :interval_previous [String, Symbol] - # @param :interval [String, Symbol] - # @param :operation [Symbol] - # @param :dimensions [Hash] - # @param :beginning [Time] - # @return [Rollup] - def square_rollup(name:, new_name:, interval:, dimensions:, beginning:, operation: :sum) - Rollup.where(name: name, interval: interval) - .where_dimensions(**dimensions) - .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) - .group(*dimensions_to_jsonb_query(dimensions)) - .rollup(new_name, interval: interval, update: true) do |rollup| - rollup.try(operation, :value) - end - end - # Reducir las estadísticas calculadas aplicando un rollup sobre el # intervalo más amplio. # diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index a0b1181e..fe50929d 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -68,11 +68,7 @@ class UriCollectionJob < PeriodicJob end # Reducir todas las visitas a cantidad de visitas por host - square_rollup(name: 'host|uri', - new_name: 'host', - interval: starting_interval, - dimensions: host_dimensions, - beginning: beginning) + rollup('host', beginning, :sum, **host_dimensions) # Acumular por mes y año reduce_rollup('host', beginning, :sum, **host_dimensions) @@ -82,12 +78,7 @@ class UriCollectionJob < PeriodicJob column_dimensions = { host: host } column_dimensions[column] = value - square_rollup(name: "host|uri|#{column}", - new_name: "host|#{column}", - interval: starting_interval, - dimensions: column_dimensions, - beginning: beginning) - + rollup("host|#{column}", beginning, :sum, **column_dimensions) reduce_rollup("host|#{column}", beginning, :sum, **column_dimensions) end end From e00780c7f0d7fee3e0364ffb1fc7cfc7ac5b237c Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 17:55:53 -0300 Subject: [PATCH 132/164] mostrar la cantidad de visitas por origen y pais --- app/controllers/stats_controller.rb | 14 ++++++++++++++ app/jobs/uri_collection_job.rb | 2 +- app/models/stat.rb | 8 ++++++++ app/views/stats/index.haml | 21 +++++++++++++++++++++ config/locales/en.yml | 11 +++++++++++ config/locales/es.yml | 11 +++++++++++ 6 files changed, 66 insertions(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 35f7f108..84ca3028 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -39,6 +39,20 @@ class StatsController < ApplicationController last_stat chart_options normalized_urls + + expires_in = Time.now.try(:"end_of_#{Stat.default_interval}") - Time.now + @columns = {} + + Stat::COLUMNS.each do |column| + @columns[column] = + Rails.cache.fetch("stats/#{column}/#{site.id}", expires_in: expires_in) do + Rollup.where(interval: interval, name: "host|#{column}") + .where_dimensions(host: hostnames) + .group("dimensions->>'#{column}'") + .sum(:value) + .transform_values(&:to_i) + end + end end # Genera un gráfico de visitas por dominio asociado a este sitio diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index fe50929d..a7dd2a00 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -26,7 +26,7 @@ class UriCollectionJob < PeriodicJob # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME # Columnas a agrupar - columns = %i[http_referer geoip2_data_country_name].zip([nil]).to_h + columns = Stat::COLUMNS.zip([nil]).to_h # Recorremos todos los hostnames y uris posibles y luego agrupamos # recursivamente para no tener que recalcular, asumiendo que es más diff --git a/app/models/stat.rb b/app/models/stat.rb index 5f72ccd0..f1016026 100644 --- a/app/models/stat.rb +++ b/app/models/stat.rb @@ -5,6 +5,14 @@ class Stat < ApplicationRecord # XXX: Los intervalos van en orden de mayor especificidad a menor INTERVALS = %i[day month year].freeze RESOURCES = %i[builds space_used build_time].freeze + COLUMNS = %i[http_referer geoip2_data_country_name].freeze belongs_to :site + + # El intervalo por defecto + # + # @return [Symbol] + def self.default_interval + INTERVALS.first + end end diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 02683781..be0c1426 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -32,6 +32,27 @@ - if @normalized_urls.present? = line_chart site_stats_uris_path(urls: @normalized_urls, **@chart_params), **@chart_options + .row.mb-5.row-cols-1.row-cols-md-2 + - @columns.each_pair do |column, values| + - next if values.blank? + .col.mb-5 + %h2= t(".columns.#{column}.title") + %p.lead= t(".columns.#{column}.description") + + %table.table + %colgroup + %col + %col + %thead + %tr.sticky-top.background-white + %th{ scope: 'col' }= t(".columns.#{column}.column") + %th{ scope: 'col' }= t('.columns.visits') + %tfoot + %tbody + - values.each_pair do |col, val| + %tr + %th{ scope: 'row', style: 'word-break: break-all' }= col.blank? ? t('.columns.empty') : col + %td= val .mb-5 %h2= t('.resources.title') %p.lead= t('.resources.description') diff --git a/config/locales/en.yml b/config/locales/en.yml index 732eb4bd..76760958 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -284,6 +284,17 @@ en: build_time: title: 'Publication time' description: 'Average time your site takes to build.' + columns: + visits: "Visits" + empty: "(nothing)" + http_referer: + title: "Referers" + description: "Visits by origin" + column: "Referer" + geoip2_data_country_name: + title: "Countries" + description: "Visits by country" + column: "Country" sites: donations: url: 'https://donaciones.sutty.nl/en/' diff --git a/config/locales/es.yml b/config/locales/es.yml index e47c1fb5..82bfd253 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -289,6 +289,17 @@ es: build_time: title: 'Tiempo de publicación' description: 'Tiempo promedio que toma en publicarse tu sitio.' + columns: + visits: "Visitas" + empty: "(nada)" + http_referer: + title: "Referencias" + description: "Orígenes de las visitas" + column: "Referencia" + geoip2_data_country_name: + title: "Países" + description: "Cantidad de visitas por país" + column: "País" sites: donations: url: 'https://donaciones.sutty.nl/' From 224e599de2f9b5acf1ae75ebfc5463b37364d179 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 18:05:08 -0300 Subject: [PATCH 133/164] fixup! mostrar la cantidad de visitas por origen y pais --- app/jobs/uri_collection_job.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index a7dd2a00..c3e48ecd 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -68,7 +68,7 @@ class UriCollectionJob < PeriodicJob end # Reducir todas las visitas a cantidad de visitas por host - rollup('host', beginning, :sum, **host_dimensions) + rollup('host', beginning, **host_dimensions) # Acumular por mes y año reduce_rollup('host', beginning, :sum, **host_dimensions) @@ -78,7 +78,7 @@ class UriCollectionJob < PeriodicJob column_dimensions = { host: host } column_dimensions[column] = value - rollup("host|#{column}", beginning, :sum, **column_dimensions) + rollup("host|#{column}", beginning, **column_dimensions) reduce_rollup("host|#{column}", beginning, :sum, **column_dimensions) end end From 99e3f093fcd1fda9447ebcfefd6661e6053f76a4 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 18:20:45 -0300 Subject: [PATCH 134/164] ordenar de menor a mayor --- app/controllers/stats_controller.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 84ca3028..e00b9822 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -51,6 +51,9 @@ class StatsController < ApplicationController .group("dimensions->>'#{column}'") .sum(:value) .transform_values(&:to_i) + .sort_by do |_, v| + v.last + end.reverse.to_h end end end From 09696529411acdbf190ad45ce7fd3fbe11f7cd31 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 18:22:35 -0300 Subject: [PATCH 135/164] acomodar el valor a la cantidad de nodos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit en realidad para estas visitas no tiene mucho sentido y de hecho es en las que estaríamos perdiendo información, porque nada indica que no hubo otros países y referencias que quedaron en otros nodos... --- app/controllers/stats_controller.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index e00b9822..07755c73 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -51,6 +51,9 @@ class StatsController < ApplicationController .group("dimensions->>'#{column}'") .sum(:value) .transform_values(&:to_i) + .transform_values do |v| + v * nodes + end .sort_by do |_, v| v.last end.reverse.to_h From 8244532c5a0fa5e00a3501b376c5d0d4fe329af7 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 18:30:24 -0300 Subject: [PATCH 136/164] fixup! ordenar de menor a mayor --- app/controllers/stats_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 07755c73..36b72721 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -55,7 +55,7 @@ class StatsController < ApplicationController v * nodes end .sort_by do |_, v| - v.last + v end.reverse.to_h end end From e4009e59dcf73bf84dd9e6f340618e49799ee60d Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 18:42:50 -0300 Subject: [PATCH 137/164] las tablas con cabeceras sticky usan el color de fondo del theme --- app/assets/stylesheets/application.scss | 1 + app/views/posts/index.haml | 4 ++-- app/views/stats/index.haml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/assets/stylesheets/application.scss b/app/assets/stylesheets/application.scss index e806a032..b756759a 100644 --- a/app/assets/stylesheets/application.scss +++ b/app/assets/stylesheets/application.scss @@ -126,6 +126,7 @@ ol.breadcrumb { color: var(--foreground); } +.table tr.sticky-top, .form-control, .custom-file-label { background-color: var(--background); diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 90d65670..accfdb34 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -71,8 +71,8 @@ %table.table{ data: { controller: 'reorder' } } %caption.sr-only= t('posts.caption') %thead - %tr - %th.border-0.background-white.position-sticky{ style: 'top: 0; z-index: 2', colspan: '4' } + %tr.sticky-top + %th.border-0{ colspan: '4' } .d-flex.flex-row.justify-content-between %div = submit_tag t('posts.reorder.submit'), class: 'btn' diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index be0c1426..6bbf596e 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -44,7 +44,7 @@ %col %col %thead - %tr.sticky-top.background-white + %tr.sticky-top %th{ scope: 'col' }= t(".columns.#{column}.column") %th{ scope: 'col' }= t('.columns.visits') %tfoot From e8300364d1e49e1b5877512f8695fda955e44da3 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 19:11:57 -0300 Subject: [PATCH 138/164] pasar todas las uris a la base de datos --- app/jobs/uri_collection_job.rb | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index c3e48ecd..90789aba 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -45,25 +45,21 @@ class UriCollectionJob < PeriodicJob # que indican las páginas y recursos descargables, el resto son # imágenes, CSS, JS y tipografías que no nos aportan números # significativos. - uris.each do |uri| - break if stop? + name = 'host|uri' + dimensions = { host: host, uri: uris } - name = 'host|uri' - dimensions = { host: host, uri: uri } + rollup(name, beginning, **dimensions) + reduce_rollup(name, beginning, :sum, **dimensions) - rollup(name, beginning, **dimensions) - reduce_rollup(name, beginning, :sum, **dimensions) + columns.each_pair do |column, values| + # Obtener orígenes de visitas por host + values.each do |value| + column_name = "host|uri|#{column}" + column_dimensions = { host: host, uri: uris } + column_dimensions[column] = value - columns.each_pair do |column, values| - # Obtener orígenes de visitas por host - values.each do |value| - column_name = "host|uri|#{column}" - column_dimensions = { host: host, uri: uri } - column_dimensions[column] = value - - rollup(column_name, beginning, **column_dimensions) - reduce_rollup(column_name, beginning, :sum, **column_dimensions) - end + rollup(column_name, beginning, **column_dimensions) + reduce_rollup(column_name, beginning, :sum, **column_dimensions) end end From 26b2d34b4d031dd21a31c490ecd11bb5b3cb8064 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 19:35:14 -0300 Subject: [PATCH 139/164] =?UTF-8?q?simplificar=20el=20c=C3=B3digo!!?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit envía todos los parámetros a la base de datos para acelerar por un montón la generación de datos --- app/jobs/concerns/recursive_rollup.rb | 5 +- app/jobs/stat_collection_job.rb | 8 ++-- app/jobs/uri_collection_job.rb | 69 ++++++++++----------------- 3 files changed, 33 insertions(+), 49 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index 4d9e11b0..1da14c01 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -30,17 +30,16 @@ module RecursiveRollup # intervalo más amplio. # # @param :name [String] - # @param :beginning [Time] # @param :operation [Symbol] # @param :dimensions [Hash] # @return [nil] - def reduce_rollup(name, beginning, operation, **dimensions) + def reduce_rollup(name:, dimensions:, operation: :sum) Stat::INTERVALS.reduce do |previous, current| recursive_rollup(name: name, interval_previous: previous, interval: current, dimensions: dimensions, - beginning: beginning, + beginning: beginning_of_interval, operation: operation) # Devolver el intervalo actual diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 63d44999..e402e3b5 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -22,9 +22,11 @@ class StatCollectionJob < PeriodicJob rollup.average(:seconds) end - reduce_rollup('builds', beginning, :sum, site_id: site_id) - reduce_rollup('space_used', beginning, :average, site_id: site_id) - reduce_rollup('build_time', beginning, :average, site_id: site_id) + dimensions = { site_id: site_id } + + reduce_rollup(name: 'builds', operation: :sum, dimensions: dimensions) + reduce_rollup(name: 'space_used', operation: :average, dimensions: dimensions) + reduce_rollup(name: 'build_time', operation: :average, dimensions: dimensions) stat.touch run_again! unless once diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 90789aba..a8c51d84 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -22,7 +22,7 @@ class UriCollectionJob < PeriodicJob @site = Site.find site_id # Obtener el principio del intervalo anterior - beginning = beginning_of_interval + beginning_of_interval # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME # Columnas a agrupar @@ -33,51 +33,33 @@ class UriCollectionJob < PeriodicJob # rápido buscar en los rollups indexados que en la tabla en bruto. # # Los referers solo se agrupan por host. - site.hostnames.each do |host| - break if stop? + columns.each_key do |column| + columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) + end - host_dimensions = { host: host } - columns.each_key do |column| - columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) - end + # Las URIs son la fuente de verdad de las visitas, porque son las + # que indican las páginas y recursos descargables, el resto son + # imágenes, CSS, JS y tipografías que no nos aportan números + # significativos. + uri_dimensions = { host: site.hostnames, uri: uris } + host_dimensions = { host: site.hostnames } - # Las URIs son la fuente de verdad de las visitas, porque son las - # que indican las páginas y recursos descargables, el resto son - # imágenes, CSS, JS y tipografías que no nos aportan números - # significativos. - name = 'host|uri' - dimensions = { host: host, uri: uris } + # Cantidad de visitas por host + rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) + reduce_rollup(name: 'host', operation: :sum, dimensions: uri_dimensions) - rollup(name, beginning, **dimensions) - reduce_rollup(name, beginning, :sum, **dimensions) + # Cantidad de visitas por página/recurso + rollup(name: 'host|uri', dimensions: uri_dimensions) + reduce_rollup(name: 'host|uri', operation: :sum, dimensions: uri_dimensions) - columns.each_pair do |column, values| - # Obtener orígenes de visitas por host - values.each do |value| - column_name = "host|uri|#{column}" - column_dimensions = { host: host, uri: uris } - column_dimensions[column] = value + # Cantidad de visitas host y parámetro + columns.each_pair do |column, values| + column_name = "host|#{column}" + column_dimensions = { host: site.hostnames } + column_dimensions[column] = values - rollup(column_name, beginning, **column_dimensions) - reduce_rollup(column_name, beginning, :sum, **column_dimensions) - end - end - - # Reducir todas las visitas a cantidad de visitas por host - rollup('host', beginning, **host_dimensions) - - # Acumular por mes y año - reduce_rollup('host', beginning, :sum, **host_dimensions) - - columns.each_pair do |column, values| - values.each do |value| - column_dimensions = { host: host } - column_dimensions[column] = value - - rollup("host|#{column}", beginning, **column_dimensions) - reduce_rollup("host|#{column}", beginning, :sum, **column_dimensions) - end - end + rollup(name: column_name, dimensions: column_dimensions, filter: uri_dimensions) + reduce_rollup(name: column_name, dimensions: column_dimensions) end stat.touch @@ -92,9 +74,10 @@ class UriCollectionJob < PeriodicJob # @param :name [String] # @param :beginning [Time] # @param :dimensions [Hash] + # @param :filter [Hash] # @return [nil] - def rollup(name, beginning, **dimensions) - AccessLog.where(**dimensions) + def rollup(name:, beginning:, dimensions:, filter: nil) + AccessLog.where(**(filter || dimensions)) .where('created_at >= ?', beginning) .completed_requests .non_robots From 6464f35625f26ef1627682ad97ca4a143d0d8179 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 19:38:32 -0300 Subject: [PATCH 140/164] =?UTF-8?q?fixup!=20simplificar=20el=20c=C3=B3digo?= =?UTF-8?q?!!?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index a8c51d84..bab85d0d 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -28,6 +28,13 @@ class UriCollectionJob < PeriodicJob # Columnas a agrupar columns = Stat::COLUMNS.zip([nil]).to_h + # Las URIs son la fuente de verdad de las visitas, porque son las + # que indican las páginas y recursos descargables, el resto son + # imágenes, CSS, JS y tipografías que no nos aportan números + # significativos. + uri_dimensions = { host: site.hostnames, uri: uris } + host_dimensions = { host: site.hostnames } + # Recorremos todos los hostnames y uris posibles y luego agrupamos # recursivamente para no tener que recalcular, asumiendo que es más # rápido buscar en los rollups indexados que en la tabla en bruto. @@ -37,13 +44,6 @@ class UriCollectionJob < PeriodicJob columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) end - # Las URIs son la fuente de verdad de las visitas, porque son las - # que indican las páginas y recursos descargables, el resto son - # imágenes, CSS, JS y tipografías que no nos aportan números - # significativos. - uri_dimensions = { host: site.hostnames, uri: uris } - host_dimensions = { host: site.hostnames } - # Cantidad de visitas por host rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) reduce_rollup(name: 'host', operation: :sum, dimensions: uri_dimensions) From 8a3c9a76a72d2e384005f81af0013f3dad54e23e Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 19:44:02 -0300 Subject: [PATCH 141/164] =?UTF-8?q?fixup!=20fixup!=20simplificar=20el=20c?= =?UTF-8?q?=C3=B3digo!!?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index bab85d0d..ca755ca1 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -76,9 +76,9 @@ class UriCollectionJob < PeriodicJob # @param :dimensions [Hash] # @param :filter [Hash] # @return [nil] - def rollup(name:, beginning:, dimensions:, filter: nil) + def rollup(name:, dimensions:, filter: nil) AccessLog.where(**(filter || dimensions)) - .where('created_at >= ?', beginning) + .where('created_at >= ?', beginning_of_interval) .completed_requests .non_robots .group(*dimensions.keys) From 158c2244490fc79467d915e4841b6d54a34e4a49 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 20:40:54 -0300 Subject: [PATCH 142/164] generar los rollups para los siguientes intervalos --- app/jobs/uri_collection_job.rb | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index ca755ca1..0ea468da 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -11,8 +11,6 @@ # Los hostnames de un sitio van a poder obtenerse a partir de # Site#hostnames con la garantía de que son únicos. class UriCollectionJob < PeriodicJob - include RecursiveRollup - # Ignoramos imágenes porque suelen ser demasiadas y no aportan a las # estadísticas. IMAGES = %w[.png .jpg .jpeg .gif .webp .jfif].freeze @@ -76,13 +74,29 @@ class UriCollectionJob < PeriodicJob # @param :dimensions [Hash] # @param :filter [Hash] # @return [nil] - def rollup(name:, dimensions:, filter: nil) + def rollup(name:, dimensions:, interval: starting_interval, filter: nil) AccessLog.where(**(filter || dimensions)) .where('created_at >= ?', beginning_of_interval) .completed_requests .non_robots .group(*dimensions.keys) - .rollup(name, interval: starting_interval, update: true) + .rollup(name, interval: interval, update: true) + end + + # Generar rollups con el resto de la información + # + # @param :name [String] + # @param :dimensions [Hash] + # @param :filter [Hash] + # @return [nil] + def reduce_rollup(name:, dimensions:, filter: nil) + Stat::INTERVALS.reduce do |_previous, current| + rollup(name: name, dimensions: dimensions, filter: filter, interval: current) + + current + end + + nil end def stat_name From 3f29d87acdb91d59f79fd38f105ea038c1909a94 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 20:41:38 -0300 Subject: [PATCH 143/164] =?UTF-8?q?las=20tablas=20tambi=C3=A9n=20est=C3=A1?= =?UTF-8?q?n=20filtradas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 36b72721..c2eb0425 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -46,17 +46,17 @@ class StatsController < ApplicationController Stat::COLUMNS.each do |column| @columns[column] = Rails.cache.fetch("stats/#{column}/#{site.id}", expires_in: expires_in) do - Rollup.where(interval: interval, name: "host|#{column}") - .where_dimensions(host: hostnames) - .group("dimensions->>'#{column}'") - .sum(:value) - .transform_values(&:to_i) - .transform_values do |v| - v * nodes - end - .sort_by do |_, v| - v - end.reverse.to_h + rollup_scope.where(interval: interval, name: "host|#{column}") + .where_dimensions(host: hostnames) + .group("dimensions->>'#{column}'") + .sum(:value) + .transform_values(&:to_i) + .transform_values do |v| + v * nodes + end + .sort_by do |_, v| + v + end.reverse.to_h end end end From c8ff6de9ae5ab32ce58b754cbd449b9fb5182ce7 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 20:42:10 -0300 Subject: [PATCH 144/164] trasladar los parametros a los graficos --- app/controllers/stats_controller.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index c2eb0425..03d5fc9b 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -34,7 +34,14 @@ class StatsController < ApplicationController def index breadcrumb I18n.t('stats.index.title'), '' - @chart_params = { interval: interval } + params.with_defaults! default_url_options + + @chart_params = { + interval: interval, + period_start: params[:period_start], + period_end: params[:period_end] + } + hostnames last_stat chart_options From 62bdfff98c5ef174e8c7ed9246aee1fbd44f9f4d Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 20:42:38 -0300 Subject: [PATCH 145/164] poder ver y filtrar por rangos --- app/views/stats/index.haml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 6bbf596e..8987560f 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -9,9 +9,13 @@ %time{ datetime: @last_stat.updated_at } #{time_ago_in_words @last_stat.updated_at}. - .mb-5 + %form.mb-5.form-inline{ method: 'get' } - Stat::INTERVALS.each do |interval| - = link_to t(".#{interval}"), site_stats_path(interval: interval, urls: params[:urls]), class: "btn #{'btn-primary active' if @interval == interval}" + = link_to t(".#{interval}"), site_stats_path(interval: interval, urls: params[:urls], period_start: params[:period_start].to_date.try(:"beginning_of_#{interval}").to_date, period_end: params[:period_end]), class: "mb-0 btn #{'btn-primary active' if @interval == interval}" + + %input.form-control{ type: 'date', name: :period_start, value: params[:period_start] } + %input.form-control{ type: 'date', name: :period_end, value: params[:period_end] } + %button.btn.mb-0{ type: 'submit' }= t('.filter') .mb-5 %h2= t('.host.title', count: @hostnames.size) From 8ca5085aae92c72492a6ff6380f130aa0edc0012 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 20:48:53 -0300 Subject: [PATCH 146/164] fixup! generar los rollups para los siguientes intervalos --- app/jobs/uri_collection_job.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 0ea468da..42c05963 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -44,11 +44,11 @@ class UriCollectionJob < PeriodicJob # Cantidad de visitas por host rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) - reduce_rollup(name: 'host', operation: :sum, dimensions: uri_dimensions) + reduce_rollup(name: 'host', dimensions: uri_dimensions) # Cantidad de visitas por página/recurso rollup(name: 'host|uri', dimensions: uri_dimensions) - reduce_rollup(name: 'host|uri', operation: :sum, dimensions: uri_dimensions) + reduce_rollup(name: 'host|uri', dimensions: uri_dimensions) # Cantidad de visitas host y parámetro columns.each_pair do |column, values| From 35d16b11397c5ae5ee3a442cd30827080a7c4717 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 21:21:58 -0300 Subject: [PATCH 147/164] fixup! fixup! generar los rollups para los siguientes intervalos --- app/jobs/uri_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 42c05963..59f753ec 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -44,7 +44,7 @@ class UriCollectionJob < PeriodicJob # Cantidad de visitas por host rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) - reduce_rollup(name: 'host', dimensions: uri_dimensions) + reduce_rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) # Cantidad de visitas por página/recurso rollup(name: 'host|uri', dimensions: uri_dimensions) From 755db601a161a5cba148ec6567959fa2c6b1dace Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 May 2022 13:41:37 -0300 Subject: [PATCH 148/164] ordenar en la base de datos --- app/controllers/stats_controller.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 03d5fc9b..cccdb8b6 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -56,14 +56,12 @@ class StatsController < ApplicationController rollup_scope.where(interval: interval, name: "host|#{column}") .where_dimensions(host: hostnames) .group("dimensions->>'#{column}'") + .order('sum(value) desc') .sum(:value) .transform_values(&:to_i) .transform_values do |v| v * nodes end - .sort_by do |_, v| - v - end.reverse.to_h end end end From 844ba75d08b213c42e128ea6578c91d91b8369e7 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 30 May 2022 15:10:52 -0300 Subject: [PATCH 149/164] =?UTF-8?q?indicar=20qu=C3=A9=20significa=20lo=20v?= =?UTF-8?q?ac=C3=ADo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/stats/index.haml | 2 +- config/locales/en.yml | 3 ++- config/locales/es.yml | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 8987560f..058f17ef 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -55,7 +55,7 @@ %tbody - values.each_pair do |col, val| %tr - %th{ scope: 'row', style: 'word-break: break-all' }= col.blank? ? t('.columns.empty') : col + %th{ scope: 'row', style: 'word-break: break-all' }= col.blank? ? t(".columns.#{column}.empty") : col %td= val .mb-5 %h2= t('.resources.title') diff --git a/config/locales/en.yml b/config/locales/en.yml index ec8a5792..87a18ff6 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -288,15 +288,16 @@ en: description: 'Average time your site takes to build.' columns: visits: "Visits" - empty: "(nothing)" http_referer: title: "Referers" description: "Visits by origin" column: "Referer" + empty: "(direct visit)" geoip2_data_country_name: title: "Countries" description: "Visits by country" column: "Country" + empty: "(couldn't detect country)" sites: donations: url: 'https://donaciones.sutty.nl/en/' diff --git a/config/locales/es.yml b/config/locales/es.yml index 30ea15fc..3c9d11b5 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -293,15 +293,16 @@ es: description: 'Tiempo promedio que toma en publicarse tu sitio.' columns: visits: "Visitas" - empty: "(nada)" http_referer: title: "Referencias" description: "Orígenes de las visitas" column: "Referencia" + empty: "(visita directa)" geoip2_data_country_name: title: "Países" description: "Cantidad de visitas por país" column: "País" + empty: "(no se pudo detectar el país)" sites: donations: url: 'https://donaciones.sutty.nl/' From 472085301960d6829aa4c1d195cda4bc96fcac78 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 11 Jun 2022 19:33:02 -0300 Subject: [PATCH 150/164] =?UTF-8?q?al=20recargar=20la=20p=C3=A1gina=20volv?= =?UTF-8?q?er=20al=20gr=C3=A1fico?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/stats/index.haml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 058f17ef..49fbd023 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -22,10 +22,10 @@ %p.lead= t('.host.description') = line_chart site_stats_host_path(@chart_params), **@chart_options - .mb-5 + #custom-urls.mb-5 %h2= t('.urls.title') %p.lead= t('.urls.description') - %form + %form{ method: 'get', action: '#custom-urls' } %input{ type: 'hidden', name: 'interval', value: @interval } .form-group %label{ for: 'urls' }= t('.urls.label') From 6f45756e6d6b5ea12386288821fd45d6b4ebed1b Mon Sep 17 00:00:00 2001 From: f Date: Sat, 11 Jun 2022 19:33:21 -0300 Subject: [PATCH 151/164] =?UTF-8?q?mejorar=20el=20mensaje=20de=20tiempo=20?= =?UTF-8?q?de=20compilaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/locales/en.yml | 2 +- config/locales/es.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index 87a18ff6..f89778ed 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -285,7 +285,7 @@ en: description: 'Average storage space used by your site.' build_time: title: 'Publication time' - description: 'Average time your site takes to build.' + description: 'Average time your site takes to build, from pressing "Publish changes" to actually being available on your site.' columns: visits: "Visits" http_referer: diff --git a/config/locales/es.yml b/config/locales/es.yml index 3c9d11b5..bf3e93f0 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -290,7 +290,7 @@ es: description: 'Espacio en disco que ocupa en promedio tu sitio.' build_time: title: 'Tiempo de publicación' - description: 'Tiempo promedio que toma en publicarse tu sitio.' + description: 'Tiempo que tarda el sitio en generarse, desde que usas el botón "Publicar cambios" hasta que los puedes ver en el sitio' columns: visits: "Visitas" http_referer: From 00f303cf20f8b7338e402eac0d9baf8465b822e5 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 11 Jun 2022 19:42:55 -0300 Subject: [PATCH 152/164] =?UTF-8?q?procesar=20las=20estad=C3=ADsticas=20au?= =?UTF-8?q?tom=C3=A1ticamente?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Procfile | 1 + lib/tasks/stats.rake | 11 +++++++++++ monit.conf | 5 +++++ 3 files changed, 17 insertions(+) create mode 100644 lib/tasks/stats.rake diff --git a/Procfile b/Procfile index b308ffd5..8f6c7741 100644 --- a/Procfile +++ b/Procfile @@ -5,3 +5,4 @@ blazer_1h: bundle exec rake blazer:run_checks SCHEDULE="1 hour" blazer_1d: bundle exec rake blazer:run_checks SCHEDULE="1 day" blazer: bundle exec rake blazer:send_failing_checks prometheus: bundle exec prometheus_exporter -b 0.0.0.0 --prefix "sutty_" +stats: bundle exec rake stats:process_all diff --git a/lib/tasks/stats.rake b/lib/tasks/stats.rake new file mode 100644 index 00000000..9461782a --- /dev/null +++ b/lib/tasks/stats.rake @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +namespace :stats do + desc 'Process stats' + task process_all: :environment do + Site.all.pluck(:id).each do |site_id| + UriCollectionJob.perform_now site_id: site_id, once: true + StatCollectionJob.perform_now site_id: site_id, once: true + end + end +end diff --git a/monit.conf b/monit.conf index 28db0cf0..83d17449 100644 --- a/monit.conf +++ b/monit.conf @@ -30,3 +30,8 @@ check program access_logs with path "/srv/http/bin/access_logs" as uid "app" and gid "www-data" every "0 0 * * *" if status != 0 then alert + +check program stats + with path "/usr/bin/foreman run -f /srv/Procfile -d /srv stats" as uid "rails" gid "www-data" + every "0 1 * * *" + if status != 0 then alert From dd34d176ea53cd24b157ce3aaf5da238f9c47217 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 15 Jun 2022 13:34:46 -0300 Subject: [PATCH 153/164] sincronizar links duros --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index b2ffba7c..996f8cdd 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -84,7 +84,7 @@ class DeployRsync < Deploy # # @return [Boolean] def rsync - run %(rsync -avi --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -aviH --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) end # El origen es el destino de la compilación From 24c0161aac6451afc65f7f222bc4acbe00b11d2e Mon Sep 17 00:00:00 2001 From: f Date: Wed, 29 Jun 2022 18:17:23 -0300 Subject: [PATCH 154/164] =?UTF-8?q?no=20fallar=20si=20los=20layouts=20no?= =?UTF-8?q?=20tienen=20traducci=C3=B3n=20#6841?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/site.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/site.rb b/app/models/site.rb index 7d4875e5..58947263 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -166,8 +166,10 @@ class Site < ApplicationRecord end # Similar a site.i18n en jekyll-locales + # + # @return [Hash] def i18n - data[I18n.locale.to_s] + data[I18n.locale.to_s] || {} end # Devuelve el idioma por defecto del sitio, el primero de la lista. From 655c6b65d59c37be310c4c48f0b4c2cad7939a9c Mon Sep 17 00:00:00 2001 From: f Date: Fri, 1 Jul 2022 14:43:30 -0300 Subject: [PATCH 155/164] procesar todas las urls a veces las urls vienen como arrays, con lo que no hay que procesarlas. --- app/controllers/stats_controller.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index cccdb8b6..c2c7bc58 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -133,7 +133,8 @@ class StatsController < ApplicationController def normalized_urls @normalized_urls ||= begin - urls = params.permit(:urls).try(:[], :urls)&.split("\n")&.map(&:strip)&.select(&:present?)&.select do |uri| + urls = params[:urls].is_a?(Array) ? params[:urls] : params[:urls]&.split("\n") + urls = urls&.map(&:strip)&.select(&:present?)&.select do |uri| uri.start_with? 'https://' end From e6c03df13f9bf7d0d6e27e1fbc7cecb54cd280f0 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 1 Jul 2022 15:01:57 -0300 Subject: [PATCH 156/164] =?UTF-8?q?mostrar=20las=20estadisticas=20en=20el?= =?UTF-8?q?=20men=C3=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/posts/index.haml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index accfdb34..c6ffc2f5 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -21,6 +21,9 @@ - if policy(@site).private? = link_to t('sites.private'), '../private/' + @site.name, class: 'btn', target: '_blank', rel: 'noopener' + - if policy(SiteStat.new(@site)).index? + = link_to t('stats.index.title'), site_stats_path(@site), class: 'btn' + - if policy(SiteUsuarie.new(@site, current_usuarie)).index? = render 'layouts/btn_with_tooltip', tooltip: t('usuaries.index.help.self'), From f1137c6ed28c5245f27865c489904a1012f119b3 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 1 Jul 2022 15:02:17 -0300 Subject: [PATCH 157/164] error de sintaxis --- config/application.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/config/application.rb b/config/application.rb index 7c530e69..f1bf8365 100644 --- a/config/application.rb +++ b/config/application.rb @@ -38,6 +38,7 @@ module Sutty config.active_storage.variant_processor = :vips + config.to_prepare do # Load application's model / class decorators Dir.glob(File.join(File.dirname(__FILE__), '..', 'app', '**', '*_decorator.rb')).sort.each do |c| Rails.configuration.cache_classes ? require(c) : load(c) From 2a416b4e670fc8690538a2de70d501b72e0ee436 Mon Sep 17 00:00:00 2001 From: Nulo Date: Wed, 13 Jul 2022 21:43:44 +0000 Subject: [PATCH 158/164] editor: No reportar error cuando no se selecciona un archivo para subir --- app/javascript/editor/types/multimedia.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/javascript/editor/types/multimedia.ts b/app/javascript/editor/types/multimedia.ts index 2af9643a..29d2a0ab 100644 --- a/app/javascript/editor/types/multimedia.ts +++ b/app/javascript/editor/types/multimedia.ts @@ -137,8 +137,10 @@ export function setupAuxiliaryToolbar(editor: Editor): void { "click", (event) => { const files = editor.toolbar.auxiliary.multimedia.fileEl.files; - if (!files || !files.length) - throw new Error("no hay archivos para subir"); + if (!files || !files.length) { + console.info("no hay archivos para subir"); + return; + } const file = files[0]; const selectedEl = editor.contentEl.querySelector( From 332ae66b8a54636b9bafea7f992719deb6af8122 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 15 Jul 2022 13:19:08 -0300 Subject: [PATCH 159/164] =?UTF-8?q?no=20juntar=20estad=C3=ADsticas=20por?= =?UTF-8?q?=20mes=20y=20a=C3=B1o=20todav=C3=ADa?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit la forma en que las estamos recopilando es rápida pero no acumulativa, entonces los números siempre dan mal. --- app/models/stat.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/stat.rb b/app/models/stat.rb index f1016026..6c681aa4 100644 --- a/app/models/stat.rb +++ b/app/models/stat.rb @@ -3,7 +3,7 @@ # Registran cuándo fue la última recolección de datos. class Stat < ApplicationRecord # XXX: Los intervalos van en orden de mayor especificidad a menor - INTERVALS = %i[day month year].freeze + INTERVALS = %i[day].freeze RESOURCES = %i[builds space_used build_time].freeze COLUMNS = %i[http_referer geoip2_data_country_name].freeze From 091ceac788595edf537d81ffd929c0dc5084f230 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 15 Jul 2022 14:17:46 -0300 Subject: [PATCH 160/164] =?UTF-8?q?bot=C3=B3n=20de=20estad=C3=ADsticas=20m?= =?UTF-8?q?=C3=A1s=20arriba?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/posts_controller.rb | 2 ++ app/views/posts/index.haml | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index e6f836bb..c5dc0f54 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -35,6 +35,8 @@ class PostsController < ApplicationController # Filtrar los posts que les invitades no pueden ver @usuarie = site.usuarie? current_usuarie + + @site_stat = SiteStat.new(site) end def show diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index c6ffc2f5..bc5c826c 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -15,15 +15,15 @@ - else %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'btn btn-secondary btn-sm' + - if policy(@site_stat).index? + = link_to t('stats.index.title'), site_stats_path(@site), class: 'btn' + - if policy(@site).edit? = link_to t('sites.edit.btn', site: @site.title), edit_site_path(@site), class: 'btn' - if policy(@site).private? = link_to t('sites.private'), '../private/' + @site.name, class: 'btn', target: '_blank', rel: 'noopener' - - if policy(SiteStat.new(@site)).index? - = link_to t('stats.index.title'), site_stats_path(@site), class: 'btn' - - if policy(SiteUsuarie.new(@site, current_usuarie)).index? = render 'layouts/btn_with_tooltip', tooltip: t('usuaries.index.help.self'), From 69712a306e131e48f6c7416002e2e8f1f1cee515 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 25 Jul 2022 16:01:08 -0300 Subject: [PATCH 161/164] equiparar file con image #7076 --- app/views/posts/attributes/_file.haml | 16 +++++++++------- app/views/posts/attributes/_image.haml | 3 +-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/app/views/posts/attributes/_file.haml b/app/views/posts/attributes/_file.haml index 19175b9c..54f9f81a 100644 --- a/app/views/posts/attributes/_file.haml +++ b/app/views/posts/attributes/_file.haml @@ -1,7 +1,5 @@ .form-group - if metadata.static_file - = hidden_field_tag "#{base}[#{attribute}][path]", metadata.value['path'] - - case metadata.static_file.blob.content_type - when %r{\Avideo/} = video_tag url_for(metadata.static_file), @@ -14,13 +12,17 @@ - else = link_to t('posts.attribute_ro.file.download'), url_for(metadata.static_file) - .custom-control.custom-switch - = check_box_tag "#{base}[#{attribute}][path]", '', false, id: "#{base}_#{attribute}_destroy", class: 'custom-control-input' - = label_tag "#{base}_#{attribute}_destroy", t('posts.attributes.file.destroy'), class: 'custom-control-label' + -# Mantener el valor si no enviamos ninguna imagen + = hidden_field_tag "#{base}[#{attribute}][path]", metadata.value['path'] + -# Los archivos requeridos solo se pueden reemplazar + - unless metadata.required + .custom-control.custom-switch + = check_box_tag "#{base}[#{attribute}][path]", '', false, id: "#{base}_#{attribute}_destroy", class: 'custom-control-input' + = label_tag "#{base}_#{attribute}_destroy", t('posts.attributes.file.destroy'), class: 'custom-control-label' .custom-file = file_field(*field_name_for(base, attribute, :path), - **field_options(attribute, metadata), + **field_options(attribute, metadata, required: (metadata.required && !metadata.path?)), class: "custom-file-input #{invalid(post, attribute)}", data: { preview: "#{attribute}-preview" }) = label_tag "#{base}_#{attribute}_path", @@ -30,7 +32,7 @@ .form-group = label_tag "#{base}_#{attribute}_description", - post_label_t(attribute, :description, post: post) + post_label_t(attribute, :description, post: post, required: false) = text_field(*field_name_for(base, attribute, :description), value: metadata.value['description'], dir: dir, lang: locale, diff --git a/app/views/posts/attributes/_image.haml b/app/views/posts/attributes/_image.haml index c11f4196..f4d9bb3d 100644 --- a/app/views/posts/attributes/_image.haml +++ b/app/views/posts/attributes/_image.haml @@ -1,5 +1,5 @@ .form-group - - if metadata.uploaded? + - if metadata.static_file = image_tag url_for(metadata.static_file), alt: metadata.value['description'], class: 'img-fluid', @@ -37,4 +37,3 @@ **field_options(attribute, metadata, required: false)) = render 'posts/attribute_feedback', post: post, attribute: [attribute, :description], metadata: metadata - From 64e93506b77de088f3696a1588e5dfad2873bed8 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 3 Aug 2022 12:53:50 -0300 Subject: [PATCH 162/164] =?UTF-8?q?eliminar=20c=C3=B3digo=20que=20no=20se?= =?UTF-8?q?=20usa=20m=C3=A1s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/post/template_field.rb | 360 ------------------------------ 1 file changed, 360 deletions(-) delete mode 100644 app/models/post/template_field.rb diff --git a/app/models/post/template_field.rb b/app/models/post/template_field.rb deleted file mode 100644 index 08da933b..00000000 --- a/app/models/post/template_field.rb +++ /dev/null @@ -1,360 +0,0 @@ -# frozen_string_literal: true - -# Representa los distintos tipos de campos que pueden venir de una -# plantilla compleja -class Post - class TemplateField - attr_reader :post, :contents, :key - - STRING_VALUES = %w[string text url number email password date year - image video audio document].freeze - - # Tipo de valores que son archivos - FILE_TYPES = %w[image video audio document].freeze - - def initialize(post, key, contents) - @post = post - @key = key - @contents = contents - end - - def title - contents.dig('title') if complex? - end - - def subtitle - contents.dig('subtitle') if complex? - end - - # Obtiene el valor - def value - complex? ? contents.dig('value') : contents - end - - def max - return 0 if simple? - - contents.fetch('max', 0) - end - - def min - return 0 if simple? - - contents.fetch('min', 0) - end - - # TODO: volver elegante! - def type - return @type if @type - - if image? - @type = 'image' - elsif email? - @type = 'email' - elsif url? - @type = 'url' - elsif number? - @type = 'number' - elsif password? - @type = 'password' - elsif date? - @type = 'date' - elsif year? - @type = 'year' - elsif text_area? - @type = 'text_area' - elsif check_box_group? - @type = 'check_box_group' - elsif radio_group? - @type = 'radio_group' - elsif string? - @type = 'text' - # TODO: volver a hacer funcionar esto y ahorranos los multiple: - # false - elsif string? && contents.split('/', 2).count == 2 - @type = 'select' - elsif nested? - @type = 'table' - elsif array? - @type = 'select' - elsif boolean? - @type = 'check_box' - end - - @type - end - - # Devuelve los valores vacíos según el tipo - def empty_value - if string? - '' - elsif nested? - # TODO: devolver las keys también - {} - elsif array? - [] - elsif boolean? - false - end - end - - def cols - complex? && contents.dig('cols') - end - - def align - complex? && contents.dig('align') - end - - # El campo es requerido si es complejo y se especifica que lo sea - def required? - complex? && contents.dig('required') - end - - def boolean? - value.is_a?(FalseClass) || value.is_a?(TrueClass) - end - - def string? - value.is_a? String - end - - def text_area? - value == 'text' - end - - def url? - value == 'url' - end - - def email? - value == 'email' || value == 'mail' - end - alias mail? email? - - def date? - value == 'date' - end - - def password? - value == 'password' - end - - def number? - value == 'number' - end - - def year? - value == 'year' - end - - def file? - string? && FILE_TYPES.include?(value) - end - - def image? - array? ? value.first == 'image' : value == 'image' - end - - # Si la plantilla es simple no está admitiendo Hashes como valores - def simple? - !complex? - end - - def complex? - contents.is_a? Hash - end - - # XXX Retrocompatibilidad - def to_s - key - end - - # Convierte el campo en un parámetro - def to_param - if nested? - { key.to_sym => {} } - elsif array? && multiple? - { key.to_sym => [] } - else - key.to_sym - end - end - - # Convierte la plantilla en el formato de front_matter - def to_front_matter - { key => empty_value } - end - - def check_box_group? - array? && (complex? && contents.fetch('checkbox', false)) - end - - def radio_group? - array? && (complex? && contents.fetch('radio', false)) - end - - def array? - value.is_a? Array - end - - # TODO: detectar cuando es complejo y tomar el valor de :multiple - def multiple? - # si la plantilla es simple, es multiple cuando tenemos un array - return array? if simple? - - array? && contents.fetch('multiple', true) - end - - # Detecta si el valor es una tabla de campos - def nested? - value.is_a?(Hash) || (array? && value.first.is_a?(Hash)) - end - - # Un campo acepta valores abiertos si no es un array con múltiples - # elementos - def open? - # Todos los valores simples son abiertos - return true unless complex? - return false unless array? - - # La cosa se complejiza cuando tenemos valores complejos - # - # Si tenemos una lista cerrada de valores, necesitamos saber si el - # campo es abierto o cerrado. Si la lista tiene varios elementos, - # es una lista cerrada, opcionalmente abierta. Si la lista tiene - # un elemento, quiere decir que estamos autocompletando desde otro - # lado. - contents.fetch('open', value.count < 2) - end - - def closed? - !open? - end - - # Determina si los valores del campo serán públicos después - # - # XXX Esto es solo una indicación, el theme Jekyll tiene que - # respetarlos por su lado luego - def public? - # Todos los campos son públicos a menos que se indique lo - # contrario - simple? || contents.fetch('public', true) - end - - def private? - !public? - end - - def human - h = key.humanize - - h - end - - def label - h = (complex? && contents.dig('label')) || human - h += ' *' if required? - - h - end - - def help - complex? && contents.dig('help') - end - - def nested_fields - return unless nested? - - v = value - v = value.first if array? - - @nested_fields ||= v.map do |k, sv| - Post::TemplateField.new post, k, sv - end - end - - # Obtiene los valores posibles para el campo de la plantilla - def values - return 'false' if value == false - return 'true' if value == true - # XXX por alguna razón `value` no refiere a value() :/ - return '' if STRING_VALUES.include? value - # Las listas cerradas no necesitan mayor procesamiento - return value if array? && closed? && value.count > 1 - # Y las vacías tampoco - return value if array? && value.empty? - # Ahorrarnos el trabajo - return @values if @values - - # Duplicar el valor para no tener efectos secundarios luego (?) - value = self.value.dup - - # Para obtener los valores posibles, hay que procesar la string y - # convertirla a parametros - - # Si es una array de un solo elemento, es un indicador de que - # tenemos que rellenarla con los valores que indica. - # - # El primer valor es el que trae la string de autocompletado - values = array? ? value.shift : value - - # Si el valor es un array con más de un elemento, queremos usar - # esas opciones. Pero si además es abierto, queremos traer los - # valores cargados anteriormente. - - # Procesamos el valor, buscando : como separador de campos que - # queremos encontrar y luego los unimos - _value = (values&.split(':', 2) || []).map do |v| - # Tenemos hasta tres niveles de búsqueda - collection, attr, subattr = v.split('/', 3) - - if collection == 'site' - # TODO: puede ser peligroso permitir acceder a cualquier - # atributo de site? No estamos trayendo nada fuera de - # lo normal - post.site.send(attr.to_sym) - # Si hay un subatributo, tenemos que averiguar todos los - # valores dentro de el - # TODO volver elegante! - # TODO volver recursivo! - elsif subattr - post.site.everything_of(attr, lang: collection) - .compact - .map { |sv| sv[subattr] } - .flatten - .compact - .uniq - else - post.site.everything_of(attr, lang: collection).compact - end - end - - # Si el valor es abierto, sumar los valores auto-completados a - # lo pre-cargados. - # - # En este punto _value es un array de 1 o 2 arrays, si es de uno, - # value tambien tiene que serlo. Si es de 2, hay que unir cada - # una - if open? - if _value.count == 1 - _value = [(_value.first + value).uniq] - elsif _value.count == 2 - _value = _value.each_with_index.map do |v, i| - v + value.fetch(i, []) - end - end - end - - # Crea un array de arrays, útil para los select - # [ [ 1, a ], [ 2, b ] ] - # aunque si no hay un : en el autocompletado, el array queda - # [ [ 1, 1 ], [ 2, 2 ] ] - values = _value.empty? ? [] : _value.last.zip(_value.first) - - # En última instancia, traer el valor por defecto y ahorrarnos - # volver a procesar - @values = values - end - end -end From cacb355b620cb5547ca21ec3743aa15c5a6d8834 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 3 Aug 2022 13:24:36 -0300 Subject: [PATCH 163/164] los deploys pueden no tener destino closes #7151 closes #7152 closes #7153 closes #7154 closes #7167 closes #7168 closes #7169 closes #7170 closes #7380 closes #7381 closes #7382 closes #7383 closes #7394 closes #7395 closes #7396 closes #7397 closes #7405 closes #7406 closes #7407 closes #7408 closes #7491 closes #7492 closes #7493 closes #7494 closes #7509 closes #7510 closes #7511 closes #7512 closes #7527 closes #7528 closes #7529 closes #7530 --- app/jobs/uri_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 59f753ec..4cbbf593 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -109,7 +109,7 @@ class UriCollectionJob < PeriodicJob # # TODO: Cambiar al mergear origin-referer def destinations - @destinations ||= site.deploys.map(&:destination).select do |d| + @destinations ||= site.deploys.map(&:destination).compact.select do |d| File.directory?(d) end.map do |d| File.realpath(d) From 8dcc8596e2e9dfb2aa1c58a5dc16e8c9e2c69b1a Mon Sep 17 00:00:00 2001 From: f Date: Tue, 7 Feb 2023 09:19:38 -0300 Subject: [PATCH 164/164] feat: informar la rama actual del repositorio #9804 --- .profile | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .profile diff --git a/.profile b/.profile new file mode 100644 index 00000000..3c73ffa9 --- /dev/null +++ b/.profile @@ -0,0 +1,9 @@ +Color_Off='\e[0m' +BPurple='\e[1;35m' +BBlue='\e[1;34m' + +is_git() { + git rev-parse --abbrev-ref HEAD 2>/dev/null +} + +PS1="\[${BPurple}\]\$(is_git) \[${BBlue}\]\W\[${Color_Off}\] >_ "