From 3a2ce1d47d1d111c5afe9f7d1bfdd482ff256207 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 17:50:32 -0300 Subject: [PATCH 001/121] =?UTF-8?q?Configurar=20Blazer=20para=20mostrar=20?= =?UTF-8?q?estad=C3=ADsticas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/application_controller.rb | 9 +++++++++ config/blazer.yml | 2 +- config/routes.rb | 7 +++---- db/migrate/20200206163257_install_blazer.rb | 2 -- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index acd0134d..f95159f4 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -62,6 +62,15 @@ class ApplicationController < ActionController::Base render 'application/page_not_found', status: :not_found end + # Necesario para poder acceder a Blazer. Solo les usuaries de este + # sitio pueden acceder al panel. + def require_usuarie + unless find_site.usuarie? current_usuarie + redirect_to root_path + return + end + end + protected def configure_permitted_parameters diff --git a/config/blazer.yml b/config/blazer.yml index 2ba0965f..11792ff1 100644 --- a/config/blazer.yml +++ b/config/blazer.yml @@ -50,7 +50,7 @@ user_method: current_usuarie user_name: email # custom before_action to use for auth -# before_action_method: require_admin +before_action_method: require_usuarie # email to send checks from from_email: blazer@<%= ENV.fetch('SUTTY', 'sutty.nl') %> diff --git a/config/routes.rb b/config/routes.rb index 2c5f1c60..186dd66f 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -4,8 +4,6 @@ Rails.application.routes.draw do devise_for :usuaries get '/.well-known/change-password', to: redirect('/usuaries/edit') - mount Blazer::Engine, at: 'blazer' - root 'application#index' constraints(Constraints::ApiSubdomain.new) do @@ -38,6 +36,9 @@ Rails.application.routes.draw do match '/api/v3/projects/:site_id/notices' => 'api/v1/notices#create', via: %i[post] resources :sites, constraints: { site_id: %r{[^/]+}, id: %r{[^/]+} } do + # Usar Blazer para mostrar estadísticas + mount Blazer::Engine, at: 'stats', as: 'stats' + # Gestionar actualizaciones del sitio get 'pull', to: 'sites#fetch' post 'pull', to: 'sites#merge' @@ -73,7 +74,5 @@ Rails.application.routes.draw do # Compilar el sitio post 'enqueue', to: 'sites#enqueue' post 'reorder_posts', to: 'sites#reorder_posts' - - resources :stats, only: [:index] end end diff --git a/db/migrate/20200206163257_install_blazer.rb b/db/migrate/20200206163257_install_blazer.rb index 9b084169..32799053 100644 --- a/db/migrate/20200206163257_install_blazer.rb +++ b/db/migrate/20200206163257_install_blazer.rb @@ -3,8 +3,6 @@ # Blazer class InstallBlazer < ActiveRecord::Migration[6.0] def change - return unless Rails.env.production? - create_table :blazer_queries do |t| t.references :creator t.string :name From 7511afbf88575fdc62265d0adfe02b56ac764eb6 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 17:51:35 -0300 Subject: [PATCH 002/121] Les usuaries tienen consultas --- app/models/usuarie.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/models/usuarie.rb b/app/models/usuarie.rb index 6de7ba4b..c88dcc68 100644 --- a/app/models/usuarie.rb +++ b/app/models/usuarie.rb @@ -11,6 +11,8 @@ class Usuarie < ApplicationRecord has_many :roles has_many :sites, through: :roles + has_many :blazer_audits, foreign_key: 'user_id', class_name: 'Blazer::Audit' + has_many :blazer_queries, foreign_key: 'creator_id', class_name: 'Blazer::Query' def name email.split('@', 2).first From ddc459130a45b9aa15a590bfa276218facb99030 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 17:54:27 -0300 Subject: [PATCH 003/121] Acceder a la lista de consultas disponibles con la identidad de Sutty --- app/controllers/application_controller.rb | 7 +++++++ app/views/blazer/queries/home.haml | 9 +++++++++ app/views/layouts/_breadcrumb.haml | 2 +- app/views/layouts/blazer/application.haml | 14 ++++++++++++++ config/locales/en.yml | 6 ++++++ config/locales/es.yml | 6 ++++++ 6 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 app/views/blazer/queries/home.haml create mode 100644 app/views/layouts/blazer/application.haml diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index f95159f4..2b5088f7 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -69,6 +69,13 @@ class ApplicationController < ActionController::Base redirect_to root_path return end + + # Necesario para los breadcrumbs. + ActionView::Base.include Loaf::ViewExtensions unless ActionView::Base.included_modules.include? Loaf::ViewExtensions + + breadcrumb current_usuarie.email, main_app.edit_usuarie_registration_path + breadcrumb 'sites.index', main_app.sites_path, match: :exact + breadcrumb 'stats.index', root_path, match: :exact end protected diff --git a/app/views/blazer/queries/home.haml b/app/views/blazer/queries/home.haml new file mode 100644 index 00000000..325ba121 --- /dev/null +++ b/app/views/blazer/queries/home.haml @@ -0,0 +1,9 @@ +#queries + %table.table + %tbody.list + - @queries.each do |query| + %tr + -# + Por alguna razón no tenemos acceso a query_path para poder + generar la URL según Rails + %td= link_to query[:name], "/sites/#{params[:site_id]}/stats/queries/#{query[:to_param]}" diff --git a/app/views/layouts/_breadcrumb.haml b/app/views/layouts/_breadcrumb.haml index c4920bc7..f209787c 100644 --- a/app/views/layouts/_breadcrumb.haml +++ b/app/views/layouts/_breadcrumb.haml @@ -20,5 +20,5 @@ role: 'button', class: 'btn' %li.nav-item - = link_to t('.logout'), destroy_usuarie_session_path, + = link_to t('.logout'), main_app.destroy_usuarie_session_path, method: :delete, role: 'button', class: 'btn' diff --git a/app/views/layouts/blazer/application.haml b/app/views/layouts/blazer/application.haml new file mode 100644 index 00000000..add94190 --- /dev/null +++ b/app/views/layouts/blazer/application.haml @@ -0,0 +1,14 @@ +!!! +%html + %head + %meta{content: 'text/html; charset=UTF-8', 'http-equiv': 'Content-Type'}/ + %title= blazer_title ? blazer_title : 'Sutty' + %meta{charset: 'utf-8'}/ + = favicon_link_tag 'blazer/favicon.png' + = stylesheet_link_tag 'application' + = javascript_pack_tag 'blazer', 'data-turbolinks-track': 'reload' + = csrf_meta_tags + %body{ class: yield(:body) } + .container-fluid#sutty + = render 'layouts/breadcrumb' + = yield diff --git a/config/locales/en.yml b/config/locales/en.yml index fc194eab..7fd83f63 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -576,3 +576,9 @@ en: edit: 'Editing' usuaries: index: 'Users' + stats: + index: 'Statistics' + blazer: + queries: + show: + empty: '(empty)' diff --git a/config/locales/es.yml b/config/locales/es.yml index e8185391..0a48742a 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -584,3 +584,9 @@ es: edit: 'Editando' usuaries: index: 'Usuaries' + stats: + index: 'Estadísticas' + blazer: + queries: + show: + empty: '(vacío)' From 49e5603687af77ac18d20a1f2975baed18e98925 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 17:57:40 -0300 Subject: [PATCH 004/121] Por cuidados, modificar Blazer pero no poder crear consultas --- app/controllers/concerns/blazer_decorator.rb | 44 ++++++++++++++++++++ config/application.rb | 7 ++++ 2 files changed, 51 insertions(+) create mode 100644 app/controllers/concerns/blazer_decorator.rb diff --git a/app/controllers/concerns/blazer_decorator.rb b/app/controllers/concerns/blazer_decorator.rb new file mode 100644 index 00000000..99b162e6 --- /dev/null +++ b/app/controllers/concerns/blazer_decorator.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# Modificaciones para Blazer +module BlazerDecorator + # No poder obtener información de la base de datos. + module DisableDatabaseInfo + extend ActiveSupport::Concern + + included do + def docs; end + + def tables; end + + def schema; end + end + end + + # Deshabilitar edición de consultas y chequeos. + module DisableEdits + extend ActiveSupport::Concern + + included do + def create; end + + def update; end + + def destroy; end + + def run; end + + def refresh; end + + def cancel; end + end + end +end + +classes = [Blazer::QueriesController, Blazer::ChecksController, Blazer::DashboardsController] +modules = [BlazerDecorator::DisableDatabaseInfo, BlazerDecorator::DisableEdits] +classes.each do |klass| + modules.each do |modul| + klass.include modul unless klass.included_modules.include? modul + end +end diff --git a/config/application.rb b/config/application.rb index 7326ae0f..5b6e373c 100644 --- a/config/application.rb +++ b/config/application.rb @@ -38,6 +38,13 @@ module Sutty config.active_storage.variant_processor = :vips + config.to_prepare do + # Load application's model / class decorators + Dir.glob(File.join(File.dirname(__FILE__), '../app/**/*_decorator.rb')) do |c| + Rails.configuration.cache_classes ? require(c) : load(c) + end + end + config.after_initialize do ActiveStorage::DirectUploadsController.include ActiveStorage::AuthenticatedDirectUploadsController From 85ad518d8d4e92fe0254b43e8775b7c0f2b76830 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 18:01:34 -0300 Subject: [PATCH 005/121] =?UTF-8?q?Correr=20Blazer=20sincr=C3=B3nicamente?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deshabilitamos la funcionalidad asincrónica de Blazer porque ejecuta unos malabares extraños con JS, que filtran la consulta hacia el código. La idea es ejecutar consultas más livianas, con lo que por ahora no lo necesitamos. Podríamos recuperar esto usando ActionCable luego. Además, reimplementa la lógica de generación de gráficos en el controlador, para simplificar la vista. --- app/controllers/concerns/blazer_decorator.rb | 145 +++++++++++++++++++ app/views/blazer/queries/show.haml | 51 +++++++ package.json | 2 + yarn.lock | 35 ++++- 4 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 app/views/blazer/queries/show.haml diff --git a/app/controllers/concerns/blazer_decorator.rb b/app/controllers/concerns/blazer_decorator.rb index 99b162e6..f2598573 100644 --- a/app/controllers/concerns/blazer_decorator.rb +++ b/app/controllers/concerns/blazer_decorator.rb @@ -33,6 +33,149 @@ module BlazerDecorator def cancel; end end end + + # Blazer hace un gran esfuerzo para ejecutar consultas de forma + # asincrónica pero termina enviándolas por JS. + module RunSync + extend ActiveSupport::Concern + + included do + alias_method :original_show, :show + + include Blazer::BaseHelper + + def show + original_show + + options = { user: blazer_user, query: @query, run_id: SecureRandom.uuid, async: false } + @data_source = Blazer.data_sources[@query.data_source] + @result = Blazer::RunStatement.new.perform(@data_source, @statement, options) + chart_data + end + + private + + # blazer-2.4.2/app/views/blazer/queries/run.html.erb + def chart_type + case @result.chart_type + when /\Aline(2)?\z/ + chart_options.merge! min: nil + when /\Abar(2)?\z/ + chart_options.merge! library: { tooltips: { intersect: false, axis: 'x' } } + when 'pie' + chart_options + when 'scatter' + chart_options.merge! library: { tooltips: { intersect: false } }, xtitle: @result.columns[0], + ytitle: @result.columns[1] + when nil + else + if @result.column_types.size == 2 + chart_options.merge! library: { tooltips: { intersect: false, axis: 'x' } } + else + chart_options.merge! library: { tooltips: { intersect: false } } + end + end + + @result.chart_type + end + + def chart_data + @chart_data ||= + case chart_type + when 'line' + @result.columns[1..-1].each_with_index.map do |k, i| + { + name: blazer_series_name(k), + data: @result.rows.map do |r| + [r[0], r[i + 1]] + end, + library: series_library[i] + } + end + when 'line2' + @result.rows.group_by do |r| + v = r[1] + (@result.boom[@result.columns[1]] || {})[v.to_s] || v + end.each_with_index.map do |(name, v), i| + { + name: blazer_series_name(name), + data: v.map do |v2| + [v2[0], v2[2]] + end, + library: series_library[i] + } + end + when 'pie' + @result.rows.map do |r| + [(@result.boom[@result.columns[0]] || {})[r[0].to_s] || r[0], r[1]] + end + when 'bar' + (@result.rows.first.size - 1).times.map do |i| + name = @result.columns[i + 1] + + { + name: blazer_series_name(name), + data: @result.rows.first(20).map do |r| + [(@result.boom[@result.columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]] + end + } + end + when 'bar2' + first_20 = @result.rows.group_by { |r| r[0] }.values.first(20).flatten(1) + labels = first_20.map { |r| r[0] }.uniq + series = first_20.map { |r| r[1] }.uniq + labels.each do |l| + series.each do |s| + first_20 << [l, s, 0] unless first_20.find { |r| r[0] == l && r[1] == s } + end + end + + first_20.group_by do |r| + v = r[1] + (@result.boom[@result.columns[1]] || {})[v.to_s] || v + end.each_with_index.map do |(name, v), _i| + { + name: blazer_series_name(name), + data: v.sort_by do |r2| + labels.index(r2[0]) + end.map do |v2| + v3 = v2[0] + [(@result.boom[@result.columns[0]] || {})[v3.to_s] || v3, v2[2]] + end + } + end + when 'scatter' + @result.rows + end + end + + def target_index + @target_index ||= @result.columns.index do |k| + k.downcase == 'target' + end + end + + def series_library + @series_library ||= {}.tap do |sl| + if target_index + color = '#109618' + sl[target_index - 1] = { + pointStyle: 'line', + hitRadius: 5, + borderColor: color, + pointBackgroundColor: color, + backgroundColor: color, + pointHoverBackgroundColor: color + } + end + end + end + + def chart_options + @chart_options ||= { id: SecureRandom.hex } + end + end + end end classes = [Blazer::QueriesController, Blazer::ChecksController, Blazer::DashboardsController] @@ -42,3 +185,5 @@ classes.each do |klass| klass.include modul unless klass.included_modules.include? modul end end + +Blazer::QueriesController.include BlazerDecorator::RunSync diff --git a/app/views/blazer/queries/show.haml b/app/views/blazer/queries/show.haml new file mode 100644 index 00000000..5ec3da98 --- /dev/null +++ b/app/views/blazer/queries/show.haml @@ -0,0 +1,51 @@ +- blazer_title @query.name +.container + .row + .col-12 + %h1= @query.name + - if @query.description.present? + %p.lead= @query.description + - unless @result.chart_type.blank? + .col-12 + - case @result.chart_type + - when 'line' + = line_chart @chart_data, **@chart_options + - when 'line2' + = line_chart @chart_data, **@chart_options + - when 'pie' + = pie_chart @chart_data, **@chart_options + - when 'bar' + = column_chart @chart_data, **@chart_options + - when 'bar2' + = column_chart @chart_data, **@chart_options + - when 'scatter' + = scatter_chart @chart_data, **@chart_options + .col-12 + %table.table + %thead + %tr + - @result.columns.each do |key| + - next if key.include? 'ciphertext' + - next if key.include? 'encrypted' + %th.position-sticky.background-white= key + %tbody + - @result.rows.each do |row| + %tr + - row.each_with_index do |v, i| + - k = @result.columns[i] + - next if k.include? 'ciphertext' + - next if k.include? 'encrypted' + %td + - if v.is_a?(Time) + - v = blazer_time_value(@data_source, k, v) + + - unless v.nil? + - if v.is_a?(String) && v.empty? + %span.text-muted= t('.empty') + - elsif @data_source.linked_columns[k] + = link_to blazer_format_value(k, v), @data_source.linked_columns[k].gsub('{value}', u(v.to_s)), target: '_blank' + - else + = blazer_format_value(k, v) + + - if (v2 = (@result.boom[k] || {})[v.nil? ? v : v.to_s]) + %span.text-muted= v2 diff --git a/package.json b/package.json index 0a2458a6..6340651f 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,8 @@ "@rails/ujs": "^6.1.3-1", "@rails/webpacker": "5.2.1", "babel-loader": "^8.2.2", + "chart.js": "2.9.3", + "chartkick": "3.2.1", "circular-dependency-plugin": "^5.2.2", "commonmark": "^0.29.0", "fork-awesome": "^1.1.7", diff --git a/yarn.lock b/yarn.lock index 11ff78cb..b478ff67 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2119,6 +2119,34 @@ chalk@^4.1.0: ansi-styles "^4.1.0" supports-color "^7.1.0" +chart.js@2.9.3: + version "2.9.3" + resolved "https://registry.yarnpkg.com/chart.js/-/chart.js-2.9.3.tgz#ae3884114dafd381bc600f5b35a189138aac1ef7" + integrity sha512-+2jlOobSk52c1VU6fzkh3UwqHMdSlgH1xFv9FKMqHiNCpXsGPQa/+81AFa+i3jZ253Mq9aAycPwDjnn1XbRNNw== + dependencies: + chartjs-color "^2.1.0" + moment "^2.10.2" + +chartjs-color-string@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/chartjs-color-string/-/chartjs-color-string-0.6.0.tgz#1df096621c0e70720a64f4135ea171d051402f71" + integrity sha512-TIB5OKn1hPJvO7JcteW4WY/63v6KwEdt6udfnDE9iCAZgy+V4SrbSxoIbTw/xkUIapjEI4ExGtD0+6D3KyFd7A== + dependencies: + color-name "^1.0.0" + +chartjs-color@^2.1.0: + version "2.4.1" + resolved "https://registry.yarnpkg.com/chartjs-color/-/chartjs-color-2.4.1.tgz#6118bba202fe1ea79dd7f7c0f9da93467296c3b0" + integrity sha512-haqOg1+Yebys/Ts/9bLo/BqUcONQOdr/hoEr2LLTRl6C5LXctUdHxsCYfvQVg5JIxITrfCNUDr4ntqmQk9+/0w== + dependencies: + chartjs-color-string "^0.6.0" + color-convert "^1.9.3" + +chartkick@3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/chartkick/-/chartkick-3.2.1.tgz#a80c2005ae353c5ae011d0a756b6f592fc8fc7a9" + integrity sha512-zV0kUeZNqrX28AmPt10QEDXHKadbVFOTAFkCMyJifHzGFkKzGCDXxVR8orZ0fC1HbePzRn5w6kLCOVxDQbMUCg== + chokidar@^2.1.8: version "2.1.8" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.8.tgz#804b3a7b6a99358c3c5c61e71d8728f041cff917" @@ -2238,7 +2266,7 @@ collection-visit@^1.0.0: map-visit "^1.0.0" object-visit "^1.0.0" -color-convert@^1.9.0, color-convert@^1.9.1: +color-convert@^1.9.0, color-convert@^1.9.1, color-convert@^1.9.3: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== @@ -5005,6 +5033,11 @@ mkdirp@^1.0.3, mkdirp@^1.0.4: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== +moment@^2.10.2: + version "2.29.1" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3" + integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ== + move-concurrently@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92" From 75e6b7a8014ea83362e126dc85dcd681c053514d Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 18:25:42 -0300 Subject: [PATCH 006/121] =?UTF-8?q?Agregar=20los=20helpers=20de=20Devise?= =?UTF-8?q?=20tambi=C3=A9n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Para que tengamos `current_usuarie` dentro de las vistas de Blazer. --- app/controllers/application_controller.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 2b5088f7..efe1a4a3 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -71,7 +71,9 @@ class ApplicationController < ActionController::Base end # Necesario para los breadcrumbs. - ActionView::Base.include Loaf::ViewExtensions unless ActionView::Base.included_modules.include? Loaf::ViewExtensions + [Loaf::ViewExtensions, Devise::Controllers::UrlHelpers].each do |helper_module| + ActionView::Base.include helper_module unless ActionView::Base.included_modules.include? helper_module + end breadcrumb current_usuarie.email, main_app.edit_usuarie_registration_path breadcrumb 'sites.index', main_app.sites_path, match: :exact From 841279f6cf81d545f064e94dc3aab39e7d2cc454 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 18:31:17 -0300 Subject: [PATCH 007/121] =?UTF-8?q?fixup!=20Agregar=20los=20helpers=20de?= =?UTF-8?q?=20Devise=20tambi=C3=A9n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/application_controller.rb | 4 +--- app/views/layouts/_breadcrumb.haml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index efe1a4a3..2b5088f7 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -71,9 +71,7 @@ class ApplicationController < ActionController::Base end # Necesario para los breadcrumbs. - [Loaf::ViewExtensions, Devise::Controllers::UrlHelpers].each do |helper_module| - ActionView::Base.include helper_module unless ActionView::Base.included_modules.include? helper_module - end + ActionView::Base.include Loaf::ViewExtensions unless ActionView::Base.included_modules.include? Loaf::ViewExtensions breadcrumb current_usuarie.email, main_app.edit_usuarie_registration_path breadcrumb 'sites.index', main_app.sites_path, match: :exact diff --git a/app/views/layouts/_breadcrumb.haml b/app/views/layouts/_breadcrumb.haml index f209787c..dc0e3158 100644 --- a/app/views/layouts/_breadcrumb.haml +++ b/app/views/layouts/_breadcrumb.haml @@ -12,7 +12,7 @@ - else %span.line-clamp-1= link_to crumb.name, crumb.url - - if current_usuarie + - if @current_usuarie || current_usuarie %ul.navbar-nav - if @site&.tienda? %li.nav-item From af67c39dc477581fb6a0acd2adce5aacb683d0aa Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 18:37:15 -0300 Subject: [PATCH 008/121] Agregar el nombre del sitio en la miga de pan --- app/controllers/application_controller.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 2b5088f7..c9e5a999 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -65,7 +65,8 @@ class ApplicationController < ActionController::Base # Necesario para poder acceder a Blazer. Solo les usuaries de este # sitio pueden acceder al panel. def require_usuarie - unless find_site.usuarie? current_usuarie + site = find_site + unless site.usuarie? current_usuarie redirect_to root_path return end @@ -75,6 +76,7 @@ class ApplicationController < ActionController::Base breadcrumb current_usuarie.email, main_app.edit_usuarie_registration_path breadcrumb 'sites.index', main_app.sites_path, match: :exact + breadcrumb site.title, main_app.site_path(site), match: :exact breadcrumb 'stats.index', root_path, match: :exact end From 44450da5208b55582c2ff287c538d71a452da274 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 Aug 2021 18:41:09 -0300 Subject: [PATCH 009/121] Les usuaries solo pueden ver sus propias consultas. --- app/controllers/concerns/blazer_decorator.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/controllers/concerns/blazer_decorator.rb b/app/controllers/concerns/blazer_decorator.rb index f2598573..eee0db64 100644 --- a/app/controllers/concerns/blazer_decorator.rb +++ b/app/controllers/concerns/blazer_decorator.rb @@ -55,6 +55,11 @@ module BlazerDecorator private + # Solo mostrar las consultas de le usuarie + def set_queries(_) + @queries = (@current_usuarie || current_usuarie).blazer_queries + end + # blazer-2.4.2/app/views/blazer/queries/run.html.erb def chart_type case @result.chart_type From 6f08ca6c36b1297d93bd7ba435e5e7d70b930a60 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 3 Aug 2021 10:10:30 -0300 Subject: [PATCH 010/121] =?UTF-8?q?A=20veces=20se=20pasa=20el=20par=C3=A1m?= =?UTF-8?q?etro=20en=20set=5Fqueries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/concerns/blazer_decorator.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/concerns/blazer_decorator.rb b/app/controllers/concerns/blazer_decorator.rb index eee0db64..876f423d 100644 --- a/app/controllers/concerns/blazer_decorator.rb +++ b/app/controllers/concerns/blazer_decorator.rb @@ -56,7 +56,7 @@ module BlazerDecorator private # Solo mostrar las consultas de le usuarie - def set_queries(_) + def set_queries(_ = nil) @queries = (@current_usuarie || current_usuarie).blazer_queries end From 489cbb414ccd4a7fbd5c81f7e6051838a4512ada Mon Sep 17 00:00:00 2001 From: f Date: Tue, 3 Aug 2021 10:15:48 -0300 Subject: [PATCH 011/121] Ya no usamos un hash de queries --- app/views/blazer/queries/home.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/blazer/queries/home.haml b/app/views/blazer/queries/home.haml index 325ba121..977b6bda 100644 --- a/app/views/blazer/queries/home.haml +++ b/app/views/blazer/queries/home.haml @@ -6,4 +6,4 @@ -# Por alguna razón no tenemos acceso a query_path para poder generar la URL según Rails - %td= link_to query[:name], "/sites/#{params[:site_id]}/stats/queries/#{query[:to_param]}" + %td= link_to query[:name], "/sites/#{params[:site_id]}/stats/queries/#{query.to_param}" From e87fad33eae8b1225d090a9e6f5464933c69dea6 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 3 Aug 2021 10:16:26 -0300 Subject: [PATCH 012/121] =?UTF-8?q?La=20tabla=20se=20puede=20scrollear=20y?= =?UTF-8?q?=20mantiene=20los=20t=C3=ADtulos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/blazer/queries/show.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/blazer/queries/show.haml b/app/views/blazer/queries/show.haml index 5ec3da98..3b2beed6 100644 --- a/app/views/blazer/queries/show.haml +++ b/app/views/blazer/queries/show.haml @@ -27,7 +27,7 @@ - @result.columns.each do |key| - next if key.include? 'ciphertext' - next if key.include? 'encrypted' - %th.position-sticky.background-white= key + %th.position-sticky.background-white{ style: 'top: 0' }= key %tbody - @result.rows.each do |row| %tr From 71ff9e5e7b22013b5c4f033518871ea9670d52c5 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 3 Aug 2021 10:24:18 -0300 Subject: [PATCH 013/121] =?UTF-8?q?Traducir=20columnas=20o=20mostrarlas=20?= =?UTF-8?q?como=20t=C3=ADtulos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Los nombres más comunas de columnas se pueden agregar al archivo de traducción, si la traducción no existe se convierte a un título. ```ruby "total_horas".titleize => "Total Horas" ``` --- app/views/blazer/queries/show.haml | 2 +- config/locales/en.yml | 5 +++++ config/locales/es.yml | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/app/views/blazer/queries/show.haml b/app/views/blazer/queries/show.haml index 3b2beed6..3b5cb152 100644 --- a/app/views/blazer/queries/show.haml +++ b/app/views/blazer/queries/show.haml @@ -27,7 +27,7 @@ - @result.columns.each do |key| - next if key.include? 'ciphertext' - next if key.include? 'encrypted' - %th.position-sticky.background-white{ style: 'top: 0' }= key + %th.position-sticky.background-white{ style: 'top: 0' }= t("blazer.columns.#{key}", default: key.titleize) %tbody - @result.rows.each do |row| %tr diff --git a/config/locales/en.yml b/config/locales/en.yml index 7fd83f63..18faa8bb 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -579,6 +579,11 @@ en: stats: index: 'Statistics' blazer: + columns: + total: 'Total' + dia: 'Date' + date: 'Date' + visitas: 'Visits' queries: show: empty: '(empty)' diff --git a/config/locales/es.yml b/config/locales/es.yml index 0a48742a..5229a591 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -587,6 +587,11 @@ es: stats: index: 'Estadísticas' blazer: + columns: + total: 'Total' + dia: 'Fecha' + date: 'Fecha' + visitas: 'Visitas' queries: show: empty: '(vacío)' From 71436d3be49cb3d0cdde8e37338369f10fdaf81f Mon Sep 17 00:00:00 2001 From: f Date: Wed, 4 Aug 2021 12:17:49 -0300 Subject: [PATCH 014/121] =?UTF-8?q?Usar=20el=20sistema=20de=20autorizaci?= =?UTF-8?q?=C3=B3n=20de=20Sutty?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Y eliminar código sin utilizar. --- app/controllers/application_controller.rb | 9 ++++----- app/controllers/posts_controller.rb | 3 --- app/controllers/private_controller.rb | 2 -- app/controllers/sites_controller.rb | 3 --- app/controllers/stats_controller.rb | 18 ------------------ app/models/{site_stat.rb => site_blazer.rb} | 2 +- app/policies/site_blazer_policy.rb | 10 ++++++++++ app/policies/site_stat_policy.rb | 15 --------------- 8 files changed, 15 insertions(+), 47 deletions(-) delete mode 100644 app/controllers/stats_controller.rb rename app/models/{site_stat.rb => site_blazer.rb} (50%) create mode 100644 app/policies/site_blazer_policy.rb delete mode 100644 app/policies/site_stat_policy.rb diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index c9e5a999..d8498218 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -3,6 +3,7 @@ # Forma de ingreso a Sutty class ApplicationController < ActionController::Base include ExceptionHandler + include Pundit protect_from_forgery with: :null_session, prepend: true @@ -10,6 +11,7 @@ class ApplicationController < ActionController::Base before_action :configure_permitted_parameters, if: :devise_controller? around_action :set_locale + rescue_from Pundit::NilPolicyError, with: :page_not_found rescue_from ActionController::RoutingError, with: :page_not_found rescue_from ActionController::ParameterMissing, with: :page_not_found @@ -33,7 +35,7 @@ class ApplicationController < ActionController::Base def find_site id = params[:site_id] || params[:id] - unless (site = current_usuarie.sites.find_by_name(id)) + unless (site = current_usuarie&.sites&.find_by_name(id)) raise SiteNotFound end @@ -66,10 +68,7 @@ class ApplicationController < ActionController::Base # sitio pueden acceder al panel. def require_usuarie site = find_site - unless site.usuarie? current_usuarie - redirect_to root_path - return - end + authorize SiteBlazer.new(site) # Necesario para los breadcrumbs. ActionView::Base.include Loaf::ViewExtensions unless ActionView::Base.included_modules.include? Loaf::ViewExtensions diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index 3ef26720..448592de 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -2,9 +2,6 @@ # Controlador para artículos class PostsController < ApplicationController - include Pundit - rescue_from Pundit::NilPolicyError, with: :page_not_found - before_action :authenticate_usuarie! # TODO: Traer los comunes desde ApplicationController diff --git a/app/controllers/private_controller.rb b/app/controllers/private_controller.rb index bb4d782d..01b6888c 100644 --- a/app/controllers/private_controller.rb +++ b/app/controllers/private_controller.rb @@ -6,8 +6,6 @@ class PrivateController < ApplicationController # XXX: Permite ejecutar JS skip_forgery_protection - include Pundit - # Enviar el archivo si existe, agregar una / al final siempre para no # romper las direcciones relativas. def show diff --git a/app/controllers/sites_controller.rb b/app/controllers/sites_controller.rb index bdaa9011..b4826226 100644 --- a/app/controllers/sites_controller.rb +++ b/app/controllers/sites_controller.rb @@ -2,9 +2,6 @@ # Controlador de sitios class SitesController < ApplicationController - include Pundit - rescue_from Pundit::NilPolicyError, with: :page_not_found - before_action :authenticate_usuarie! breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb deleted file mode 100644 index 07baaf1a..00000000 --- a/app/controllers/stats_controller.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -# Estadísticas del sitio -class StatsController < ApplicationController - include Pundit - before_action :authenticate_usuarie! - - def index - @site = find_site - authorize SiteStat.new(@site) - - # Solo queremos el promedio de tiempo de compilación, no de - # instalación de dependencias. - stats = @site.build_stats.jekyll - @build_avg = stats.average(:seconds).to_f.round(2) - @build_max = stats.maximum(:seconds).to_f.round(2) - end -end diff --git a/app/models/site_stat.rb b/app/models/site_blazer.rb similarity index 50% rename from app/models/site_stat.rb rename to app/models/site_blazer.rb index 73503aca..76dee12a 100644 --- a/app/models/site_stat.rb +++ b/app/models/site_blazer.rb @@ -1,3 +1,3 @@ # frozen_string_literal: true -SiteStat = Struct.new(:site) +SiteBlazer = Struct.new(:site) diff --git a/app/policies/site_blazer_policy.rb b/app/policies/site_blazer_policy.rb new file mode 100644 index 00000000..a6ea01b7 --- /dev/null +++ b/app/policies/site_blazer_policy.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +# Les invitades no pueden ver las estadísticas (aun) +SiteBlazerPolicy = Struct.new(:usuarie, :site_blazer) do + def home? + site_blazer&.site&.usuarie? usuarie + end + + alias_method :show?, :home? +end diff --git a/app/policies/site_stat_policy.rb b/app/policies/site_stat_policy.rb deleted file mode 100644 index a797034c..00000000 --- a/app/policies/site_stat_policy.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -# Política de acceso a las estadísticas -class SiteStatPolicy - attr_reader :site_stat, :usuarie - - def initialize(usuarie, site_stat) - @usuarie = usuarie - @site_stat = site_stat - end - - def index? - site_stat.site.usuarie? usuarie - end -end From 249b115af848f02fb11dd1bbebd55c3a31f5fa28 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 4 Aug 2021 20:16:56 -0300 Subject: [PATCH 015/121] No enviar URLs en los chequeos. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Necesita más trabajo hacer esto y no tenemos un uso inmediato para chequeos específicos de sitios. fixes #2331 --- .../blazer/check_mailer/failing_checks.haml | 5 ++++ .../blazer/check_mailer/state_change.haml | 30 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 app/views/blazer/check_mailer/failing_checks.haml create mode 100644 app/views/blazer/check_mailer/state_change.haml diff --git a/app/views/blazer/check_mailer/failing_checks.haml b/app/views/blazer/check_mailer/failing_checks.haml new file mode 100644 index 00000000..8624ae9b --- /dev/null +++ b/app/views/blazer/check_mailer/failing_checks.haml @@ -0,0 +1,5 @@ +%ul + - @checks.each do |check| + %li + = link_to check.query.name + = check.state diff --git a/app/views/blazer/check_mailer/state_change.haml b/app/views/blazer/check_mailer/state_change.haml new file mode 100644 index 00000000..48418a58 --- /dev/null +++ b/app/views/blazer/check_mailer/state_change.haml @@ -0,0 +1,30 @@ +!!! +%html + %head + %meta{:content => "text/html; charset=UTF-8", "http-equiv" => "Content-Type"}/ + %body{:style => "font-family: 'Helvetica Neue', Arial, Helvetica; font-size: 14px; color: #333;"} + - if @error + %p= @error + - elsif @rows_count > 0 && @check_type == "bad_data" + %p + - if @rows_count <= 10 + = pluralize(@rows_count, "row") + - else + Showing 10 of #{@rows_count} rows + %table{:style => "width: 100%; border-spacing: 0; border-collapse: collapse;"} + %thead + %tr + - @columns.first(5).each do |column| + %th{:style => "padding: 8px; line-height: 1.4; text-align: left; vertical-align: bottom; border-bottom: 2px solid #ddd; width: #{(100 / @columns.size).round(2)}%;"} + = column + %tbody + - @rows.first(10).each do |row| + %tr + - @columns.first(5).each_with_index do |column, i| + %td{:style => "padding: 8px; line-height: 1.4; vertical-align: top; border-top: 1px solid #ddd;"} + - value = row[i] + - if @column_types[i] == "time" && value.to_s.length > 10 + - value = Time.parse(value).in_time_zone(Blazer.time_zone) rescue value + = value + - if @columns.size > 5 + %p{:style => "color: #999;"} Only first 5 columns shown From 5885dd7e9615288bbe36cf0b413c3aeda042853e Mon Sep 17 00:00:00 2001 From: f Date: Fri, 6 Aug 2021 17:47:27 -0300 Subject: [PATCH 016/121] fixup! No enviar URLs en los chequeos. --- app/views/blazer/check_mailer/failing_checks.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/blazer/check_mailer/failing_checks.haml b/app/views/blazer/check_mailer/failing_checks.haml index 8624ae9b..c28c3936 100644 --- a/app/views/blazer/check_mailer/failing_checks.haml +++ b/app/views/blazer/check_mailer/failing_checks.haml @@ -1,5 +1,5 @@ %ul - @checks.each do |check| %li - = link_to check.query.name + = check.query.name = check.state From accb559f014ccc287ffcc5e6f15ce5ddae918a53 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 16:39:56 -0300 Subject: [PATCH 017/121] hacer deploys remotos --- app/models/deploy_rsync.rb | 88 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 app/models/deploy_rsync.rb diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb new file mode 100644 index 00000000..40018dbd --- /dev/null +++ b/app/models/deploy_rsync.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +# Sincroniza sitios a servidores remotos usando Rsync. El servidor +# remoto tiene que tener rsync instalado. +class DeployRsync < Deploy + store :values, accessors: %i[flags destination host_keys], coder: JSON + + def deploy + ssh? && rsync + end + + # No se ocupa espacio local + # + # @return [Integer] + def size + 0 + end + + private + + # Verificar la conexión SSH implementando Trust On First Use + # + # TODO: Medir el tiempo que tarda en iniciarse la conexión + # + # @return [Boolean] + def ssh? + user, host = user_host + ssh_available = false + + Net::SSH.start(host, user, verify_host_key: tofu) do |ssh| + if values[:host_keys].blank? + # Guardar las llaves que se encontraron en la primera conexión + values[:host_keys] = ssh.transport.host_keys.map do |host_key| + "#{host_key.ssh_type} #{host_key.fingerprint}" + end + + ssh_available = save + else + ssh_available = true + end + end + + ssh_available + rescue Exception => e + ExceptionNotifier.notify_exception(e, data: { site: site.id, hostname: host, user: user }) + + false + end + + # Confiar en la primera llave que encontremos, fallar si cambian + # + # @return [Symbol] + def tofu + values[:host_keys].present? ? :always : :accept_new + end + + # Devuelve el par user host + # + # @return [Array] + def user_host + destination.split(':', 2).first.split('@', 2).tap do |d| + next unless d.size == 1 + + d.insert(0, nil) + end + end + + # Sincroniza hacia el directorio remoto, usando las flags opcionales. + # + # @return [Boolean] + def rsync + run %(rsync -av #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + end + + # El origen es el destino de la compilación + # + # @return [String] + def source + site.deploys.find_by(type: 'DeployLocal').destination + end + + # Devolver el destino o lanzar un error si no está configurado + def destination + values[:destination].tap do |_d| + raise ArgumentError, 'destination no está configurado' + end + end +end From f10b65173a9542500e3d206a2167fb3524b59ed0 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 17:05:51 -0300 Subject: [PATCH 018/121] fixup! hacer deploys remotos --- app/models/deploy_rsync.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 40018dbd..9e855f57 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -81,8 +81,8 @@ class DeployRsync < Deploy # Devolver el destino o lanzar un error si no está configurado def destination - values[:destination].tap do |_d| - raise ArgumentError, 'destination no está configurado' + values[:destination].tap do |d| + raise(ArgumentError, 'destination no está configurado') if d.blank? end end end From 6fcdeb52f381bbde75b5156671f228016209f6ea Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 17:23:42 -0300 Subject: [PATCH 019/121] entorno --- app/models/deploy_rsync.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 9e855f57..4191b20d 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -47,6 +47,14 @@ class DeployRsync < Deploy false end + def env + { + 'HOME' => home_dir, + 'PATH' => '/usr/bin', + 'LANG' => ENV['LANG'] + } + end + # Confiar en la primera llave que encontremos, fallar si cambian # # @return [Symbol] From ea198f185b655a353755b097cdbc6c264378db34 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:07:45 -0300 Subject: [PATCH 020/121] bajar el timeout para no bloquear el deploy --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 4191b20d..2387366a 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -27,7 +27,7 @@ class DeployRsync < Deploy user, host = user_host ssh_available = false - Net::SSH.start(host, user, verify_host_key: tofu) do |ssh| + Net::SSH.start(host, user, verify_host_key: tofu, timeout: 5) do |ssh| if values[:host_keys].blank? # Guardar las llaves que se encontraron en la primera conexión values[:host_keys] = ssh.transport.host_keys.map do |host_key| From c49c63f77661413a208a47bd7a80f7b4b8d49535 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:11:21 -0300 Subject: [PATCH 021/121] timeout de rsync --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 2387366a..5d360c4c 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -77,7 +77,7 @@ class DeployRsync < Deploy # # @return [Boolean] def rsync - run %(rsync -av #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -av --timeout=5 #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) end # El origen es el destino de la compilación From 8edc9b460af993ad4c52b1192962cb548c39a82a Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:17:55 -0300 Subject: [PATCH 022/121] obtener listado de nodos de un lugar central --- app/models/sutty.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 app/models/sutty.rb diff --git a/app/models/sutty.rb b/app/models/sutty.rb new file mode 100644 index 00000000..7ec8432c --- /dev/null +++ b/app/models/sutty.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# Configuración general de Sutty +class Sutty + extend self + + # Los nodos son otros servidores de Sutty hacia los que se sincronizan + # sitios. + # + # @return [Array] + def nodes + @nodes ||= ENV.fetch('SUTTY_NODES', '').split(',') + end +end From dcbd1c02ac32c1b183d6c90255da693ad67bbc9e Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:26:41 -0300 Subject: [PATCH 023/121] =?UTF-8?q?el=20m=C3=A9todo=20es=20p=C3=BAblico?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_rsync.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 5d360c4c..50719cd1 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -16,6 +16,13 @@ class DeployRsync < Deploy 0 end + # Devolver el destino o lanzar un error si no está configurado + def destination + values[:destination].tap do |d| + raise(ArgumentError, 'destination no está configurado') if d.blank? + end + end + private # Verificar la conexión SSH implementando Trust On First Use @@ -86,11 +93,4 @@ class DeployRsync < Deploy def source site.deploys.find_by(type: 'DeployLocal').destination end - - # Devolver el destino o lanzar un error si no está configurado - def destination - values[:destination].tap do |d| - raise(ArgumentError, 'destination no está configurado') if d.blank? - end - end end From 5fac827ca09fd46231490c59f845b6ec6a2e6121 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:27:15 -0300 Subject: [PATCH 024/121] registrar los cambios --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 50719cd1..7733406e 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -84,7 +84,7 @@ class DeployRsync < Deploy # # @return [Boolean] def rsync - run %(rsync -av --timeout=5 #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -avi --timeout=5 #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) end # El origen es el destino de la compilación From 20915443d84d7ddb50ef8417d957d2a2dbb2dbcd Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:27:52 -0300 Subject: [PATCH 025/121] deprecar las flags por ahora --- app/models/deploy_rsync.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 7733406e..12f4ce0e 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -3,7 +3,7 @@ # Sincroniza sitios a servidores remotos usando Rsync. El servidor # remoto tiene que tener rsync instalado. class DeployRsync < Deploy - store :values, accessors: %i[flags destination host_keys], coder: JSON + store :values, accessors: %i[destination host_keys], coder: JSON def deploy ssh? && rsync @@ -80,11 +80,11 @@ class DeployRsync < Deploy end end - # Sincroniza hacia el directorio remoto, usando las flags opcionales. + # Sincroniza hacia el directorio remoto # # @return [Boolean] def rsync - run %(rsync -avi --timeout=5 #{flags ? Shellwords.escape(flags) : ''} #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -avi --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) end # El origen es el destino de la compilación From a5e90257a30566ecd00fc36a63bf8fcb5e789528 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:38:03 -0300 Subject: [PATCH 026/121] generar un deploy rsync para el sitio con todos los nodos --- app/services/site_service.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 5e2fc706..2ffb8cdd 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -144,4 +144,11 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do PostService.new(site: site, usuarie: usuarie, post: post, params: params).update end + + # Crea los deploys necesarios para sincronizar a otros nodos de Sutty + def sync_nodes + Sutty.nodes.each do |node| + site.deploys.build(type: 'DeployRsync', destination: "sutty@#{node}:#{site.hostname}") + end + end end From 10d950689b614f6ca2a99ace3cf14dce44b3a037 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 18:39:52 -0300 Subject: [PATCH 027/121] crear rsyncs al crear el sitio --- app/services/site_service.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 2ffb8cdd..89c5796d 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -9,6 +9,7 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do self.site = Site.new params add_role temporal: false, rol: 'usuarie' + sync_nodes I18n.with_locale(usuarie&.lang&.to_sym || I18n.default_locale) do site.save && From a0134cc052578e2ec66025b3d51b2879fc144217 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 19:05:50 -0300 Subject: [PATCH 028/121] =?UTF-8?q?ya=20existe=20un=20lugar=20donde=20guar?= =?UTF-8?q?dar=20la=20configuraci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/sutty.rb | 14 -------------- app/services/site_service.rb | 2 +- config/application.rb | 4 ++++ 3 files changed, 5 insertions(+), 15 deletions(-) delete mode 100644 app/models/sutty.rb diff --git a/app/models/sutty.rb b/app/models/sutty.rb deleted file mode 100644 index 7ec8432c..00000000 --- a/app/models/sutty.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -# Configuración general de Sutty -class Sutty - extend self - - # Los nodos son otros servidores de Sutty hacia los que se sincronizan - # sitios. - # - # @return [Array] - def nodes - @nodes ||= ENV.fetch('SUTTY_NODES', '').split(',') - end -end diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 89c5796d..22423bb8 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -148,7 +148,7 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do # Crea los deploys necesarios para sincronizar a otros nodos de Sutty def sync_nodes - Sutty.nodes.each do |node| + Rails.application.nodes.each do |node| site.deploys.build(type: 'DeployRsync', destination: "sutty@#{node}:#{site.hostname}") end end diff --git a/config/application.rb b/config/application.rb index 7326ae0f..bc948936 100644 --- a/config/application.rb +++ b/config/application.rb @@ -49,5 +49,9 @@ module Sutty EmailAddress::Config.error_messages translations.transform_keys(&:to_s), locale.to_s end end + + def nodes + @nodes ||= ENV.fetch('SUTTY_NODES', '').split(',') + end end end From aa0c359554917ef93547d3446e55ac73394587a8 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 19:06:11 -0300 Subject: [PATCH 029/121] crear un rsync por cada sitio existente --- .../20220406211042_add_deploy_rsync_to_sites.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 db/migrate/20220406211042_add_deploy_rsync_to_sites.rb diff --git a/db/migrate/20220406211042_add_deploy_rsync_to_sites.rb b/db/migrate/20220406211042_add_deploy_rsync_to_sites.rb new file mode 100644 index 00000000..92b6f17b --- /dev/null +++ b/db/migrate/20220406211042_add_deploy_rsync_to_sites.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# Agrega un DeployRsync hacia los servidores alternativos para cada +# sitio +class AddDeployRsyncToSites < ActiveRecord::Migration[6.1] + def up + Site.find_each do |site| + SiteService.new(site: site).send :sync_nodes + site.save + end + end + + def down + DeployRsync.destroy_all + end +end From f51c051af81da89345e66f137be419b00800ebe5 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 19:35:35 -0300 Subject: [PATCH 030/121] mensajes --- config/locales/en.yml | 4 ++++ config/locales/es.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/config/locales/en.yml b/config/locales/en.yml index b814796d..647f3ee8 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -100,6 +100,10 @@ en: title: Alternative domain name success: Success! error: Error + deploy_rsync: + title: Synchronize to backup server + success: Success! + error: Error help: You can contact us by replying to this e-mail maintenance_mailer: notice: diff --git a/config/locales/es.yml b/config/locales/es.yml index a6fbd407..1ccf5047 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -100,6 +100,10 @@ es: title: Dominio alternativo success: ¡Éxito! error: Hubo un error + deploy_rsync: + title: Sincronizar al servidor alternativo + success: ¡Éxito! + error: Hubo un error help: Por cualquier duda, responde este correo para contactarte con nosotres. maintenance_mailer: notice: From 8c9bd6aa88c9c584896c9109d4c04aad5d9b1cfd Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:31:49 -0300 Subject: [PATCH 031/121] por ahora no es configurable --- app/views/deploys/_deploy_rsync.haml | 1 + 1 file changed, 1 insertion(+) create mode 100644 app/views/deploys/_deploy_rsync.haml diff --git a/app/views/deploys/_deploy_rsync.haml b/app/views/deploys/_deploy_rsync.haml new file mode 100644 index 00000000..0aab9802 --- /dev/null +++ b/app/views/deploys/_deploy_rsync.haml @@ -0,0 +1 @@ +-# nada From 35e41b729ad35e86b1b4d4ce63ba2d4dc0ad97d6 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:44:46 -0300 Subject: [PATCH 032/121] el espacio ocupado es el mismo que el local --- app/models/deploy_rsync.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index 12f4ce0e..c19d279b 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -9,11 +9,11 @@ class DeployRsync < Deploy ssh? && rsync end - # No se ocupa espacio local + # El espacio remoto es el mismo que el local # # @return [Integer] def size - 0 + deploy_local.build_stats.last.size end # Devolver el destino o lanzar un error si no está configurado @@ -91,6 +91,10 @@ class DeployRsync < Deploy # # @return [String] def source - site.deploys.find_by(type: 'DeployLocal').destination + deploy_local.destination + end + + def deploy_local + @deploy_local ||= site.deploys.find_by(type: 'DeployLocal') end end From ea25d27f240cacad4fac4a1bf97455aa03c4c8f5 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:53:45 -0300 Subject: [PATCH 033/121] =?UTF-8?q?calcular=20el=20tama=C3=B1o=20a=20parti?= =?UTF-8?q?r=20de=20deploy=20local?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index c19d279b..b2ffba7c 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -13,7 +13,7 @@ class DeployRsync < Deploy # # @return [Integer] def size - deploy_local.build_stats.last.size + deploy_local.size end # Devolver el destino o lanzar un error si no está configurado From bb7c50e24b4fb9a9ce16666d53a77fc666a22506 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 6 Apr 2022 20:54:35 -0300 Subject: [PATCH 034/121] calcular el espacio una sola vez --- app/models/deploy_local.rb | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index 4fa588f5..1b661059 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -28,15 +28,17 @@ class DeployLocal < Deploy # Obtener el tamaño de todos los archivos y directorios (los # directorios son archivos :) def size - paths = [destination, File.join(destination, '**', '**')] + @size ||= begin + paths = [destination, File.join(destination, '**', '**')] - Dir.glob(paths).map do |file| - if File.symlink? file - 0 - else - File.size(file) - end - end.inject(:+) + Dir.glob(paths).map do |file| + if File.symlink? file + 0 + else + File.size(file) + end + end.inject(:+) + end end def destination From ed3f15391982e93c2902c2a33582c0ce063c5c35 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 15:06:00 -0300 Subject: [PATCH 035/121] =?UTF-8?q?es=20una=20tarea=20peri=C3=B3dica?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 2aa8d702..16f04738 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -2,7 +2,7 @@ # Genera resúmenes de información para poder mostrar estadísticas y se # corre regularmente a sí misma. -class StatCollectionJob < ApplicationJob +class StatCollectionJob < PeriodicJob STAT_NAME = 'stat_collection_job' def perform(site_id:, once: true) From 30861d240ea08e415c709c639d9b8cc87495b239 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 15:06:17 -0300 Subject: [PATCH 036/121] =?UTF-8?q?aplicar=20la=20operaci=C3=B3n!?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 16f04738..e70544c3 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -44,7 +44,7 @@ class StatCollectionJob < PeriodicJob .where_dimensions(site_id: site.id) .group("dimensions->'site_id'") .rollup(name, interval: interval, update: true) do |rollup| - rollup.try(:operation, :value) + rollup.try(operation, :value) end end From 22742cf058cef5ebf2d562b4e91ee6da2abe833f Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 15:06:55 -0300 Subject: [PATCH 037/121] migraciones faltantes --- db/migrate/20211022224008_add_site_to_stats.rb | 8 ++++++++ db/migrate/20211022225449_add_name_to_stats.rb | 9 +++++++++ 2 files changed, 17 insertions(+) create mode 100644 db/migrate/20211022224008_add_site_to_stats.rb create mode 100644 db/migrate/20211022225449_add_name_to_stats.rb diff --git a/db/migrate/20211022224008_add_site_to_stats.rb b/db/migrate/20211022224008_add_site_to_stats.rb new file mode 100644 index 00000000..db2b43ab --- /dev/null +++ b/db/migrate/20211022224008_add_site_to_stats.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +# La recolección de estadísticas podría pertenecer a un sitio +class AddSiteToStats < ActiveRecord::Migration[6.1] + def change + add_belongs_to :stats, :site, index: true, null: true + end +end diff --git a/db/migrate/20211022225449_add_name_to_stats.rb b/db/migrate/20211022225449_add_name_to_stats.rb new file mode 100644 index 00000000..89a17ee0 --- /dev/null +++ b/db/migrate/20211022225449_add_name_to_stats.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +# Agregarle un nombre a la estadística +class AddNameToStats < ActiveRecord::Migration[6.1] + def change + add_column :stats, :name, :string, null: false + add_index :stats, :name, using: 'hash' + end +end From 9b2194ee950354573fdef604a49cdafb8141d0e7 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 15:24:06 -0300 Subject: [PATCH 038/121] detener la tarea --- app/jobs/periodic_job.rb | 8 ++++++++ app/jobs/uri_collection_job.rb | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/app/jobs/periodic_job.rb b/app/jobs/periodic_job.rb index 8d9453a3..2f60a2b3 100644 --- a/app/jobs/periodic_job.rb +++ b/app/jobs/periodic_job.rb @@ -52,4 +52,12 @@ class PeriodicJob < ApplicationJob def beginning_of_interval @beginning_of_interval ||= last_stat.created_at.try(:"beginning_of_#{starting_interval}") end + + def stop_file + @stop_file ||= Rails.root.join('tmp', self.class.to_s.tableize) + end + + def stop? + File.exist? stop_file + end end diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 9ec333cd..b66a804d 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -21,7 +21,7 @@ class UriCollectionJob < PeriodicJob hostnames.each do |hostname| uris.each do |uri| - return if File.exist? Rails.root.join('tmp', 'uri_collection_job_stop') + return if stop? AccessLog.where(host: hostname, uri: uri) .where('created_at >= ?', beginning_of_interval) From ef32a6d7f6e94b266f19c35ea565db063eb3ced7 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 15:30:03 -0300 Subject: [PATCH 039/121] schema --- db/schema.rb | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/db/schema.rb b/db/schema.rb index 107e7be7..a395329d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_05_14_165639) do +ActiveRecord::Schema.define(version: 2021_10_22_225449) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -64,6 +64,7 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do t.string "remote_user" t.boolean "crawler", default: false t.string "http_referer" + t.datetime "created_at", precision: 6 t.index ["geoip2_data_city_name"], name: "index_access_logs_on_geoip2_data_city_name" t.index ["geoip2_data_country_name"], name: "index_access_logs_on_geoip2_data_country_name" t.index ["host"], name: "index_access_logs_on_host" @@ -303,6 +304,15 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do t.index ["usuarie_id"], name: "index_roles_on_usuarie_id" end + create_table "rollups", force: :cascade do |t| + t.string "name", null: false + t.string "interval", null: false + t.datetime "time", null: false + t.jsonb "dimensions", default: {}, null: false + t.float "value" + t.index ["name", "interval", "time", "dimensions"], name: "index_rollups_on_name_and_interval_and_time_and_dimensions", unique: true + end + create_table "sites", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -324,6 +334,15 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do t.index ["name"], name: "index_sites_on_name", unique: true end + create_table "stats", force: :cascade do |t| + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.bigint "site_id" + t.string "name", null: false + t.index ["name"], name: "index_stats_on_name", using: :hash + t.index ["site_id"], name: "index_stats_on_site_id" + end + create_table "usuaries", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -370,4 +389,10 @@ new.indexed_content := to_tsvector(('pg_catalog.' || new.dictionary)::regconfig, SQL_ACTIONS end + create_trigger("access_logs_before_insert_row_tr", :compatibility => 1). + on("access_logs"). + before(:insert) do + "new.created_at := to_timestamp(new.msec);" + end + end From cd1263a67211d103ccaab46b1bfe89cdc97ed536 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 15:46:24 -0300 Subject: [PATCH 040/121] usar fork por ahora --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 2b304ee0..1e476dde 100644 --- a/Gemfile +++ b/Gemfile @@ -64,7 +64,7 @@ gem 'rails-i18n' gem 'rails_warden' gem 'redis', require: %w[redis redis/connection/hiredis] gem 'redis-rails' -gem 'rollups', git: 'https://github.com/ankane/rollup.git', branch: 'master' +gem 'rollups', git: 'https://github.com/fauno/rollup.git', branch: 'update' gem 'rubyzip' gem 'rugged' gem 'concurrent-ruby-ext' From f96c7aff7d7dde7e3c874fb7f7b78e16a11c198f Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 16:28:32 -0300 Subject: [PATCH 041/121] =?UTF-8?q?volcar=20y=20eliminar=20los=20logs=20de?= =?UTF-8?q?l=20d=C3=ADa=20anterior?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/access_logs | 12 ++++++++++++ monit.conf | 5 +++++ 2 files changed, 17 insertions(+) create mode 100755 bin/access_logs diff --git a/bin/access_logs b/bin/access_logs new file mode 100755 index 00000000..2b4fd99f --- /dev/null +++ b/bin/access_logs @@ -0,0 +1,12 @@ +#!/bin/sh +set -e + +# Volcar y eliminar todos los access logs del día anterior +date=`date +%F` + +psql -h postgresql "${DATABASE:-sutty}" sutty < "/srv/http/_storage/${date}.psql.gz" +begin; +copy (select * from access_logs where created_at < '${date}') to stdout; +delete from access_logs where created_at < '${date}'; +commit; +SQL diff --git a/monit.conf b/monit.conf index f574c56d..27605d7c 100644 --- a/monit.conf +++ b/monit.conf @@ -29,3 +29,8 @@ check program blazer as uid "app" and gid "www-data" every 61 cycles if status != 0 then alert + +check program access_logs + with path "/srv/http/bin/access_logs" as uid "app" and gid "www-data" + every "0 0 * * *" + if status != 0 then alert From 83b9325d0c76e74a5cf525df45eb67b660f0246e Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 17:50:36 -0300 Subject: [PATCH 042/121] =?UTF-8?q?grabar=20la=20duraci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index b66a804d..7ca9a4d9 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -19,9 +19,12 @@ class UriCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id + # Recordar la última vez que se corrió la tarea + stat = site.stats.create! name: STAT_NAME + hostnames.each do |hostname| uris.each do |uri| - return if stop? + next if stop? AccessLog.where(host: hostname, uri: uri) .where('created_at >= ?', beginning_of_interval) @@ -46,8 +49,7 @@ class UriCollectionJob < PeriodicJob end end - # Recordar la última vez que se corrió la tarea - site.stats.create! name: STAT_NAME + stat.touch run_again! unless once end From 0cd71ad779f05923de7f1c94cc3a89e880c9aa35 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 17:54:36 -0300 Subject: [PATCH 043/121] eliminar blazer de las rutas por sitio --- config/routes.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/config/routes.rb b/config/routes.rb index c323f556..8bab18af 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -36,9 +36,6 @@ Rails.application.routes.draw do match '/api/v3/projects/:site_id/notices' => 'api/v1/notices#create', via: %i[post] resources :sites, constraints: { site_id: %r{[^/]+}, id: %r{[^/]+} } do - # Usar Blazer para mostrar estadísticas - mount Blazer::Engine, at: 'stats', as: 'stats' - # Gestionar actualizaciones del sitio get 'pull', to: 'sites#fetch' post 'pull', to: 'sites#merge' From 3edf45e27fcdd51fdf64730bc856fcf021224dc2 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 18:04:51 -0300 Subject: [PATCH 044/121] fixup! usar fork por ahora --- Gemfile.lock | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 8df2d77e..87812726 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,15 +6,6 @@ GIT rails (>= 3.0) rake (>= 0.8.7) -GIT - remote: https://github.com/ankane/rollup.git - revision: 0ab6c603450175eb1004f7793e86486943cb9f72 - branch: master - specs: - rollups (0.1.3) - activesupport (>= 5.1) - groupdate (>= 5.2) - GIT remote: https://github.com/fauno/email_address revision: 536b51f7071b68a55140c0c1726b4cd401d1c04d @@ -24,6 +15,15 @@ GIT netaddr (>= 2.0.4, < 3) simpleidn +GIT + remote: https://github.com/fauno/rollup.git + revision: ddbb345aa57e63b4cfdf7557267efa89ba60caac + branch: update + specs: + rollups (0.1.3) + activesupport (>= 5.1) + groupdate (>= 5.2) + GEM remote: https://gems.sutty.nl/ specs: @@ -214,8 +214,8 @@ GEM ffi (~> 1.0) globalid (0.6.0) activesupport (>= 5.0) - groupdate (5.2.2) - activesupport (>= 5) + groupdate (6.1.0) + activesupport (>= 5.2) hairtrigger (0.2.24) activerecord (>= 5.0, < 7) ruby2ruby (~> 2.4) From bfdd90088f76fb88560f9745218eb60e1c43ceed Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 18:29:18 -0300 Subject: [PATCH 045/121] permisos --- app/models/site_stat.rb | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 app/models/site_stat.rb diff --git a/app/models/site_stat.rb b/app/models/site_stat.rb new file mode 100644 index 00000000..73503aca --- /dev/null +++ b/app/models/site_stat.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +SiteStat = Struct.new(:site) From 0c34613cb579712045696c5d6a213c02491ddf5a Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 18:56:34 -0300 Subject: [PATCH 046/121] encontrar todos los idiomas --- app/jobs/uri_collection_job.rb | 50 ++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 7ca9a4d9..ada61bab 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -13,7 +13,7 @@ class UriCollectionJob < PeriodicJob # Ignoramos imágenes porque suelen ser demasiadas y no aportan a las # estadísticas. - IMAGES = %w[.png .jpg .jpeg .gif .webp].freeze + IMAGES = %w[.png .jpg .jpeg .gif .webp .jfif].freeze STAT_NAME = 'uri_collection_job' def perform(site_id:, once: true) @@ -91,18 +91,46 @@ class UriCollectionJob < PeriodicJob # # @return [Array] def uris - @uris ||= Dir.chdir destination do - (Dir.glob('**/*.html') + Dir.glob('public/**/*').reject do |p| - File.directory? p - end.reject do |p| - p = p.downcase + @uris ||= + locales.map do |locale| + uri = "/#{locale}/".squeeze('/') + dir = File.join(destination, locale) - IMAGES.any? do |i| - p.end_with? i + files(dir).map do |f| + uri + f end - end).map do |uri| - "/#{uri}" - end + end.flatten(2) + end + + # @return [Array] + def locales + @locales ||= ['', site.locales.map(&:to_s)].flatten(1) + end + + # @param :dir [String] + # @return [Array] + def files(dir) + Dir.chdir(dir) do + pages = Dir.glob('**/*.html') + files = Dir.glob('public/**/*') + files = remove_directories files + files = remove_images files + + [pages, files].flatten(1) + end + end + + # @param :files [Array] + # @return [Array] + def remove_directories(files) + files.reject do |f| + File.directory? f + end + end + + def remove_images(files) + files.reject do |f| + IMAGES.include? File.extname(f).downcase end end end From f94ece78cfe9c9cbf1a2ae65effed69b9000acff Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 19:06:17 -0300 Subject: [PATCH 047/121] optimizar sql usando indices --- app/jobs/uri_collection_job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index ada61bab..1bc6f768 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -37,6 +37,7 @@ class UriCollectionJob < PeriodicJob # intervalo más amplio. Stat::INTERVALS.reduce do |previous, current| Rollup.where(name: 'host|uri', interval: previous) + .where('time >= ?', beginning_of_interval) .where_dimensions(host: hostname, uri: uri) .group("dimensions->'host'", "dimensions->'uri'") .rollup('host|uri', interval: current, update: true) do |rollup| From 7cbc48d45b139c25656d8c0d8315d513d133181c Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 19:24:06 -0300 Subject: [PATCH 048/121] actualizar todo el intervalo --- app/jobs/uri_collection_job.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 1bc6f768..4d369c2d 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -36,8 +36,10 @@ class UriCollectionJob < PeriodicJob # Reducir las estadísticas calculadas aplicando un rollup sobre el # intervalo más amplio. Stat::INTERVALS.reduce do |previous, current| + beginning_of_this_interval = beginning_of_interval.try(:"beginning_of_#{current}") + Rollup.where(name: 'host|uri', interval: previous) - .where('time >= ?', beginning_of_interval) + .where('time >= ?', beginning_of_this_interval) .where_dimensions(host: hostname, uri: uri) .group("dimensions->'host'", "dimensions->'uri'") .rollup('host|uri', interval: current, update: true) do |rollup| From d8060728be781f074d9a8442408d651ef44d570c Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 19:32:55 -0300 Subject: [PATCH 049/121] =?UTF-8?q?mantener=20el=20d=C3=ADa=20anterior=20c?= =?UTF-8?q?ompleto=20en=20la=20base=20de=20datos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/access_logs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/access_logs b/bin/access_logs index 2b4fd99f..a7c151cc 100755 --- a/bin/access_logs +++ b/bin/access_logs @@ -1,8 +1,8 @@ #!/bin/sh set -e -# Volcar y eliminar todos los access logs del día anterior -date=`date +%F` +# Volcar y eliminar todos los access logs de dos días atrás +date=`bundle exec rails runner "puts (Date.today - 2.days)"` psql -h postgresql "${DATABASE:-sutty}" sutty < "/srv/http/_storage/${date}.psql.gz" begin; From 6b191d2d91541df6ef40e36a354e3a3a071b3ef5 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 19:37:08 -0300 Subject: [PATCH 050/121] =?UTF-8?q?usar=20la=20misma=20optimizaci=C3=B3n?= =?UTF-8?q?=20para=20stats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index e70544c3..3b68b61e 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -41,6 +41,7 @@ class StatCollectionJob < PeriodicJob # @return [NilClass] def rollup(name:, interval_previous:, interval:, operation: :sum) Rollup.where(name: name, interval: interval_previous) + .where('time >= ?', beginning_of_interval.try(:"beginning_of_#{interval}")) .where_dimensions(site_id: site.id) .group("dimensions->'site_id'") .rollup(name, interval: interval, update: true) do |rollup| From aad5c82cb7b7de9b673d2f4d57b41c3d39bef961 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 20:32:20 -0300 Subject: [PATCH 051/121] refactorizar rollups recursivos --- app/jobs/concerns/recursive_rollup.rb | 38 +++++++++++++ app/jobs/stat_collection_job.rb | 24 +++----- app/jobs/uri_collection_job.rb | 82 +++++++++++++++++++-------- 3 files changed, 102 insertions(+), 42 deletions(-) create mode 100644 app/jobs/concerns/recursive_rollup.rb diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb new file mode 100644 index 00000000..3163558e --- /dev/null +++ b/app/jobs/concerns/recursive_rollup.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module RecursiveRollup + extend ActiveSupport::Concern + + included do + private + + # Genera un rollup recursivo en base al período anterior y aplica una + # operación. + # + # @param :name [String] + # @param :interval_previous [String] + # @param :interval [String] + # @param :operation [Symbol] + # @param :dimensions [Hash] + # @param :new_name [String] + # @param :beginning [Time] + # @return [Rollup] + def recursive_rollup(name:, interval_previous:, interval:, dimensions:, beginning:, operation: :sum, new_name: nil) + Rollup.where(name: name, interval: interval_previous) + .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) + .where_dimensions(**dimensions) + .group(*dimensions_to_jsonb_query(dimensions)) + .rollup(new_name || name, interval: interval, update: true) do |rollup| + rollup.try(operation, :value) + end + end + + # @param :dimensions [Hash] + # @return [Array] + def dimensions_to_jsonb_query(dimensions) + dimensions.keys.map do |key| + "dimensions->'#{key}'" + end + end + end +end diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 3b68b61e..c94320d5 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -3,6 +3,8 @@ # Genera resúmenes de información para poder mostrar estadísticas y se # corre regularmente a sí misma. class StatCollectionJob < PeriodicJob + include RecursiveRollup + STAT_NAME = 'stat_collection_job' def perform(site_id:, once: true) @@ -20,9 +22,11 @@ class StatCollectionJob < PeriodicJob # XXX: Es correcto promediar promedios? Stat::INTERVALS.reduce do |previous, current| - rollup(name: 'builds', interval_previous: previous, interval: current) - rollup(name: 'space_used', interval_previous: previous, interval: current, operation: :average) - rollup(name: 'build_time', interval_previous: previous, interval: current, operation: :average) + opts = { interval_previous: previous, interval: current, dimensions: { site_id: site.id } } + + recursive_rollup(name: 'builds', **opts) + recursive_rollup(name: 'space_used', operation: :average, **opts) + recursive_rollup(name: 'build_time', operation: :average, **opts) current end @@ -35,20 +39,6 @@ class StatCollectionJob < PeriodicJob private - # Genera un rollup recursivo en base al período anterior y aplica una - # operación. - # - # @return [NilClass] - def rollup(name:, interval_previous:, interval:, operation: :sum) - Rollup.where(name: name, interval: interval_previous) - .where('time >= ?', beginning_of_interval.try(:"beginning_of_#{interval}")) - .where_dimensions(site_id: site.id) - .group("dimensions->'site_id'") - .rollup(name, interval: interval, update: true) do |rollup| - rollup.try(operation, :value) - end - end - # Los registros a procesar # # @return [ActiveRecord::Relation] diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 4d369c2d..cad05cbb 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -11,6 +11,8 @@ # Los hostnames de un sitio van a poder obtenerse a partir de # Site#hostnames con la garantía de que son únicos. class UriCollectionJob < PeriodicJob + include RecursiveRollup + # Ignoramos imágenes porque suelen ser demasiadas y no aportan a las # estadísticas. IMAGES = %w[.png .jpg .jpeg .gif .webp .jfif].freeze @@ -21,35 +23,19 @@ class UriCollectionJob < PeriodicJob # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME + name = 'host|uri' + beginning = beginning_of_interval + + hostnames.each do |host| + break if stop? - hostnames.each do |hostname| uris.each do |uri| - next if stop? + break if stop? - AccessLog.where(host: hostname, uri: uri) - .where('created_at >= ?', beginning_of_interval) - .completed_requests - .non_robots - .group(:host, :uri) - .rollup('host|uri', interval: starting_interval, update: true) - - # Reducir las estadísticas calculadas aplicando un rollup sobre el - # intervalo más amplio. - Stat::INTERVALS.reduce do |previous, current| - beginning_of_this_interval = beginning_of_interval.try(:"beginning_of_#{current}") - - Rollup.where(name: 'host|uri', interval: previous) - .where('time >= ?', beginning_of_this_interval) - .where_dimensions(host: hostname, uri: uri) - .group("dimensions->'host'", "dimensions->'uri'") - .rollup('host|uri', interval: current, update: true) do |rollup| - rollup.sum(:value) - end - - # Devolver el intervalo actual - current - end + rollup_uri(uri, host, name, beginning) end + + rollup_host(host, name, beginning) end stat.touch @@ -59,6 +45,52 @@ class UriCollectionJob < PeriodicJob private + def rollup_uri(uri, host, name, beginning) + dimensions = { host: host, uri: uri } + + AccessLog.where(**dimensions) + .where('created_at >= ?', beginning) + .completed_requests + .non_robots + .group(*dimensions.keys) + .rollup(name, interval: starting_interval, update: true) + + # Reducir las estadísticas calculadas aplicando un rollup sobre el + # intervalo más amplio. + Stat::INTERVALS.reduce do |previous, current| + recursive_rollup(name: name, + interval_previous: previous, + interval: current, + dimensions: dimensions, + beginning: beginning) + + # Devolver el intervalo actual + current + end + end + + def rollup_host(host, name, beginning) + dimensions = { host: host } + new_name = 'host' + + recursive_rollup(name: name, + new_name: new_name, + interval_previous: starting_interval, + interval: starting_interval, + dimensions: dimensions, + beginning: beginning) + + Stat::INTERVALS.reduce do |previous, current| + recursive_rollup(name: new_name, + interval_previous: previous, + interval: current, + dimensions: dimensions, + beginning: beginning) + + current + end + end + def stat_name STAT_NAME end From c09df3b87b8fa563e9225de4bed244ed4e280937 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 20:33:02 -0300 Subject: [PATCH 052/121] =?UTF-8?q?registrar=20el=20tiempo=20que=20pas?= =?UTF-8?q?=C3=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index c94320d5..03dcd292 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -10,6 +10,9 @@ class StatCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id + # Registrar que se hicieron todas las recolecciones + stat = site.stats.create! name: STAT_NAME + scope.rollup('builds', **options) scope.rollup('space_used', **options) do |rollup| @@ -31,9 +34,7 @@ class StatCollectionJob < PeriodicJob current end - # Registrar que se hicieron todas las recolecciones - site.stats.create! name: STAT_NAME - + stat.touch run_again! unless once end From 47a74b584824001e81d1bc1be86c6333ba2099b0 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 23 Apr 2022 20:33:58 -0300 Subject: [PATCH 053/121] fixup! refactorizar rollups recursivos --- app/jobs/stat_collection_job.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 03dcd292..fcb4d6e1 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -12,6 +12,7 @@ class StatCollectionJob < PeriodicJob # Registrar que se hicieron todas las recolecciones stat = site.stats.create! name: STAT_NAME + beginning = beginning_of_interval scope.rollup('builds', **options) @@ -25,7 +26,7 @@ class StatCollectionJob < PeriodicJob # XXX: Es correcto promediar promedios? Stat::INTERVALS.reduce do |previous, current| - opts = { interval_previous: previous, interval: current, dimensions: { site_id: site.id } } + opts = { interval_previous: previous, interval: current, beginning: beginning, dimensions: { site_id: site.id } } recursive_rollup(name: 'builds', **opts) recursive_rollup(name: 'space_used', operation: :average, **opts) From 298fd5bed14fa31c04dcf8d9cb5785c8c86c3c36 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:46:48 -0300 Subject: [PATCH 054/121] centralizar la forma en que se obtienen los hostnames relacionado con !50 --- app/jobs/uri_collection_job.rb | 22 +--------------------- app/models/site.rb | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index cad05cbb..ea968fa0 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -26,7 +26,7 @@ class UriCollectionJob < PeriodicJob name = 'host|uri' beginning = beginning_of_interval - hostnames.each do |host| + site.hostnames.each do |host| break if stop? uris.each do |uri| @@ -102,26 +102,6 @@ class UriCollectionJob < PeriodicJob @destination ||= site.deploys.find_by(type: 'DeployLocal').destination end - # TODO: Cambiar al mergear origin-referer - # - # @return [Array] - def hostnames - @hostnames ||= site.deploys.map do |deploy| - case deploy - when DeployLocal - site.hostname - when DeployWww - deploy.fqdn - when DeployAlternativeDomain - deploy.hostname.dup.tap do |h| - h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}") - end - when DeployHiddenService - deploy.onion - end - end.compact - end - # Recolecta todas las URIs menos imágenes # # @return [Array] diff --git a/app/models/site.rb b/app/models/site.rb index 5b78d625..bcc9e60b 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -101,6 +101,26 @@ class Site < ApplicationRecord "https://#{hostname}#{slash ? '/' : ''}" end + # TODO: Cambiar al mergear origin-referer + # + # @return [Array] + def hostnames + @hostnames ||= deploys.map do |deploy| + case deploy + when DeployLocal + hostname + when DeployWww + deploy.fqdn + when DeployAlternativeDomain + deploy.hostname.dup.tap do |h| + h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}") + end + when DeployHiddenService + deploy.onion + end + end.compact + end + # Obtiene los dominios alternativos # # @return Array @@ -123,7 +143,9 @@ class Site < ApplicationRecord # # @return Array def urls(slash: true) - alternative_urls(slash: slash) << url(slash: slash) + @urls ||= hostnames.map do |h| + "https://#{h}#{slash ? '/' : ''}" + end end def invitade?(usuarie) From fb5cc0aaa7c10749d78ce18b2246c7b20945d273 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:48:25 -0300 Subject: [PATCH 055/121] breadcrumbs --- app/controllers/stats_controller.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 44073c1f..ba9113ff 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -8,6 +8,10 @@ class StatsController < ApplicationController before_action :authenticate_usuarie! before_action :authorize_stats + breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path + breadcrumb 'sites.index', :sites_path, match: :exact + breadcrumb -> { site.title }, -> { site_posts_path(site, locale: locale) }, match: :exact + EXTRA_OPTIONS = { builds: {}, space_used: { bytes: true }, @@ -21,6 +25,8 @@ class StatsController < ApplicationController end def index + breadcrumb I18n.t('stats.index.title'), '' + @chart_params = { interval: interval } hostnames last_stat From f72d27c3083ee32b50da0baa8e948566de65e79f Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:49:26 -0300 Subject: [PATCH 056/121] =?UTF-8?q?optimizaci=C3=B3n:=20usar=20las=20dimen?= =?UTF-8?q?siones=20como=20=C3=ADndices=20y=20no=20como=20texto?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Rollup#where_dimensions` no aprovecha los índices. --- app/jobs/concerns/recursive_rollup.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index 3163558e..b3327a73 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# Implementa rollups recursivos module RecursiveRollup extend ActiveSupport::Concern @@ -18,9 +19,8 @@ module RecursiveRollup # @param :beginning [Time] # @return [Rollup] def recursive_rollup(name:, interval_previous:, interval:, dimensions:, beginning:, operation: :sum, new_name: nil) - Rollup.where(name: name, interval: interval_previous) + Rollup.where(name: name, interval: interval_previous, dimensions: dimensions.to_json) .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) - .where_dimensions(**dimensions) .group(*dimensions_to_jsonb_query(dimensions)) .rollup(new_name || name, interval: interval, update: true) do |rollup| rollup.try(operation, :value) From 7c7ae2f75041b667a8f77ecbbf43fe2c569809d8 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:52:27 -0300 Subject: [PATCH 057/121] =?UTF-8?q?mostrar=20un=20per=C3=ADodo=20por=20def?= =?UTF-8?q?ecto?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit todavía no hay interfaz para cambiarlos --- app/controllers/stats_controller.rb | 30 +++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index ba9113ff..ded3a05c 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -24,6 +24,13 @@ class StatsController < ApplicationController policy.script_src :self, :unsafe_inline end + # Parámetros por defecto + # + # @return [Hash] + def default_url_options + { interval: 'day', period_start: Date.today.beginning_of_year, period_end: Date.today } + end + def index breadcrumb I18n.t('stats.index.title'), '' @@ -36,9 +43,9 @@ class StatsController < ApplicationController # Genera un gráfico de visitas por dominio asociado a este sitio def host - return unless stale? [last_stat, hostnames, interval] + return unless stale? [last_stat, hostnames, interval, period] - stats = Rollup.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series| + stats = rollup_scope.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series| series.each do |serie| serie[:name] = serie.dig(:dimensions, 'host') serie[:data].transform_values! do |value| @@ -51,7 +58,7 @@ class StatsController < ApplicationController end def resources - return unless stale? [last_stat, interval, resource] + return unless stale? [last_stat, interval, resource, period] options = { interval: interval, @@ -60,14 +67,14 @@ class StatsController < ApplicationController } } - render json: Rollup.series(resource, **options) + render json: rollup_scope.series(resource, **options) end def uris - return unless stale? [last_stat, hostnames, interval, normalized_urls] + return unless stale? [last_stat, hostnames, interval, normalized_urls, period] options = { host: hostnames, uri: normalized_paths } - stats = Rollup.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series| + stats = rollup_scope.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series| series.each do |serie| serie[:name] = serie[:dimensions].slice('host', 'uri').values.join.sub('/index.html', '/') serie[:data].transform_values! do |value| @@ -81,6 +88,10 @@ class StatsController < ApplicationController private + def rollup_scope + Rollup.where(time: period) + end + def last_stat @last_stat ||= Stat.last end @@ -171,4 +182,11 @@ class StatsController < ApplicationController def nodes @nodes ||= ENV.fetch('NODES', 1).to_i end + + def period + @period ||= begin + p = params.permit(:period_start, :period_end) + p[:period_start]..p[:period_end] + end + end end From 3a32422e26695feb58d4c45b5526d1bca227dfb8 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:54:04 -0300 Subject: [PATCH 058/121] =?UTF-8?q?usar=20m=C3=A9todos=20del=20sitio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index ded3a05c..116bca7c 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -103,23 +103,29 @@ class StatsController < ApplicationController # TODO: Eliminar cuando mergeemos referer-origin def hostnames - @hostnames ||= [@site.hostname, @site.alternative_hostnames].flatten + @hostnames ||= site.hostnames end # Normalizar las URLs # # @return [Array] def normalized_urls - @normalized_urls ||= params.permit(:urls).try(:[], - :urls)&.split("\n")&.map(&:strip)&.select(&:present?)&.select do |uri| - uri.start_with? 'https://' - end&.map do |u| - # XXX: Eliminar - # @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1} - next u unless u.end_with? '/' + @normalized_urls ||= + begin + urls = params.permit(:urls).try(:[], :urls)&.split("\n")&.map(&:strip)&.select(&:present?)&.select do |uri| + uri.start_with? 'https://' + end - "#{u}index.html" - end&.uniq || [@site.url, @site.urls].flatten.uniq + urls ||= [site.url] + + urls.map do |u| + # XXX: Eliminar al deployear + # @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1} + next u unless u.end_with? '/' + + "#{u}index.html" + end.uniq + end end def normalized_paths From 027d24d66a83dbd00273c3310e121090ade72097 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:55:54 -0300 Subject: [PATCH 059/121] memoizar el sitio --- app/controllers/stats_controller.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 116bca7c..10b75819 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -97,8 +97,7 @@ class StatsController < ApplicationController end def authorize_stats - @site = find_site - authorize SiteStat.new(@site) + authorize SiteStat.new(site) end # TODO: Eliminar cuando mergeemos referer-origin @@ -195,4 +194,8 @@ class StatsController < ApplicationController p[:period_start]..p[:period_end] end end + + def site + @site ||= find_site + end end From 0754e29a5a272a276ebefb93ab620838e9cc2fca Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:56:25 -0300 Subject: [PATCH 060/121] =?UTF-8?q?cada=20sitio=20tiene=20su=20propio=20co?= =?UTF-8?q?ntrol=20de=20estad=C3=ADsticas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 10b75819..4933c821 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -93,7 +93,7 @@ class StatsController < ApplicationController end def last_stat - @last_stat ||= Stat.last + @last_stat ||= site.stats.last end def authorize_stats From 7e575372cf1a7a42e4817f2ebe858941163daf6a Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:56:45 -0300 Subject: [PATCH 061/121] parametros por defecto --- app/controllers/stats_controller.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 4933c821..9a8a345d 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -162,14 +162,15 @@ class StatsController < ApplicationController def interval @interval ||= begin i = params[:interval]&.to_sym - Stat::INTERVALS.include?(i) ? i : :day + Stat::INTERVALS.include?(i) ? i : Stat::INTERVALS.first end end + # @return [Symbol] def resource @resource ||= begin r = params[:resource].to_sym - Stat::RESOURCES.include?(r) ? r : :builds + Stat::RESOURCES.include?(r) ? r : Stat::RESOURCES.first end end From a703eb409643fa6fa0d068d596472e7cfff5dac3 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:57:16 -0300 Subject: [PATCH 062/121] no fallar si el sitio no existe --- app/jobs/uri_collection_job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index ea968fa0..796a289c 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -20,6 +20,7 @@ class UriCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id + return unless File.directory? destination # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME From 5186892a87c0ec15aa1a4f9cb1dafa1b5173444c Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:57:57 -0300 Subject: [PATCH 063/121] =?UTF-8?q?fixup!=20usar=20m=C3=A9todos=20del=20si?= =?UTF-8?q?tio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/stats/index.haml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index bfcf33ef..2b5d643e 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -25,12 +25,12 @@ %input{ type: 'hidden', name: 'interval', value: @interval } .form-group %label{ for: 'urls' }= t('.urls.label') - %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size, aria_describedby: 'help-urls' }= @normalized_urls.join("\n") + %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size + 1, aria_describedby: 'help-urls' }= @normalized_urls.join("\n") %small#help-urls.feedback.form-text.text-muted= t('.urls.help') .form-group %button.btn{ type: 'submit' }= t('.urls.submit') - if @normalized_urls.present? - = line_chart site_stats_uris_path(urls: params[:urls], **@chart_params), **@chart_options + = line_chart site_stats_uris_path(urls: @normalized_urls, **@chart_params), **@chart_options .mb-5 %h2= t('.resources.title') From be6224ddf173f35d45a26df68abe71aea44282f2 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:59:02 -0300 Subject: [PATCH 064/121] =?UTF-8?q?recolectar=20estad=C3=ADsticas=20de=20r?= =?UTF-8?q?eferers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 46 +++++++++++++--------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 796a289c..12f25eb6 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -24,7 +24,6 @@ class UriCollectionJob < PeriodicJob # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME - name = 'host|uri' beginning = beginning_of_interval site.hostnames.each do |host| @@ -33,10 +32,18 @@ class UriCollectionJob < PeriodicJob uris.each do |uri| break if stop? - rollup_uri(uri, host, name, beginning) + rollup('host|uri', beginning, host: host, uri: uri) + + AccessLog.where(host: host, uri: uri).distinct(:http_referer).pluck(:http_referer).each do |http_referer| + rollup('host|uri|referer', beginning, host: host, uri: uri, http_referer: http_referer) + end end - rollup_host(host, name, beginning) + rollup('host', beginning, host: host) + + AccessLog.where(host: host).distinct(:http_referer).pluck(:http_referer).each do |http_referer| + rollup('host|referer', beginning, host: host, http_referer: http_referer) + end end stat.touch @@ -46,9 +53,14 @@ class UriCollectionJob < PeriodicJob private - def rollup_uri(uri, host, name, beginning) - dimensions = { host: host, uri: uri } - + # Generar un rollup a partir de unas dimensiones que también sirven de + # filtro. + # + # @param :name [String] + # @param :beginning [Time] + # @param :dimensions [Hash] + # @return [nil] + def rollup(name, beginning, **dimensions) AccessLog.where(**dimensions) .where('created_at >= ?', beginning) .completed_requests @@ -70,28 +82,6 @@ class UriCollectionJob < PeriodicJob end end - def rollup_host(host, name, beginning) - dimensions = { host: host } - new_name = 'host' - - recursive_rollup(name: name, - new_name: new_name, - interval_previous: starting_interval, - interval: starting_interval, - dimensions: dimensions, - beginning: beginning) - - Stat::INTERVALS.reduce do |previous, current| - recursive_rollup(name: new_name, - interval_previous: previous, - interval: current, - dimensions: dimensions, - beginning: beginning) - - current - end - end - def stat_name STAT_NAME end From 3b19d8f792f743964e7b4e2a2119fcb770bb8a97 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 18:50:00 -0300 Subject: [PATCH 065/121] =?UTF-8?q?actualizaci=C3=B3n=20diaria?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/locales/en.yml | 2 +- config/locales/es.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index 639d9184..732eb4bd 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -252,7 +252,7 @@ en: help: | These statistics show information about how your site is generated and how many resources it uses. - last_update: 'Updated every hour. Last update on ' + last_update: 'Updated daily. Last update on ' empty: 'There is no enough information yet. We invite you to come back in %{please_return_at}!' loading: 'Loading...' hour: 'Hourly' diff --git a/config/locales/es.yml b/config/locales/es.yml index 86e156df..e47c1fb5 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -257,7 +257,7 @@ es: help: | Las estadísticas visibilizan información sobre cómo se genera y cuántos recursos utiliza tu sitio. - last_update: 'Actualizadas cada hora. Última actualización hace ' + last_update: 'Actualizadas diariamente. Última actualización hace ' empty: 'Todavía no hay información suficiente. Te invitamos a volver en %{please_return_at} :)' loading: 'Cargando...' hour: 'Por hora' From 479ca7430f44269b643cd26ee6684f35544a9ceb Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 18:50:11 -0300 Subject: [PATCH 066/121] nombre --- app/jobs/stat_collection_job.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index fcb4d6e1..1ee2f2b3 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -57,4 +57,8 @@ class StatCollectionJob < PeriodicJob def options @options ||= { interval: starting_interval, update: true } end + + def stat_name + STAT_NAME + end end From 752935afb414d864c8acfe28184b0984c021b38f Mon Sep 17 00:00:00 2001 From: f Date: Wed, 27 Apr 2022 13:57:33 -0300 Subject: [PATCH 067/121] calcular bien la fecha --- Dockerfile | 2 +- app/jobs/stat_collection_job.rb | 2 +- bin/access_logs | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index ee6ba871..b3fd8e78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -114,7 +114,7 @@ RUN ln -s data/_private /srv/http/_private USER root # Instalar la configuración de monit RUN install -m 640 -o root -g root /srv/http/monit.conf /etc/monit.d/sutty.conf -RUN apk add --no-cache daemonize ruby-webrick +RUN apk add --no-cache daemonize ruby-webrick dateutils RUN install -m 755 /srv/http/entrypoint.sh /usr/local/bin/sutty # Mantener estos directorios! diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 1ee2f2b3..a67a3c50 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -47,7 +47,7 @@ class StatCollectionJob < PeriodicJob def scope @scope ||= site.build_stats .jekyll - .where('created_at => ?', beginning_of_interval) + .where('created_at >= ?', beginning_of_interval) .group(:site_id) end diff --git a/bin/access_logs b/bin/access_logs index a7c151cc..cfeeb57a 100755 --- a/bin/access_logs +++ b/bin/access_logs @@ -2,9 +2,12 @@ set -e # Volcar y eliminar todos los access logs de dos días atrás -date=`bundle exec rails runner "puts (Date.today - 2.days)"` +date="`dateadd today -1d`" +file="/srv/http/_storage/${date}.psql.gz" +test -n "${date}" +test ! -f "${file}" -psql -h postgresql "${DATABASE:-sutty}" sutty < "/srv/http/_storage/${date}.psql.gz" +psql -h postgresql "${DATABASE:-sutty}" sutty < "${file}" begin; copy (select * from access_logs where created_at < '${date}') to stdout; delete from access_logs where created_at < '${date}'; From 957c810aea703d7c877ca5521fca8a9c8f9f306f Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 15:04:42 -0300 Subject: [PATCH 068/121] =?UTF-8?q?recolectar=20informaci=C3=B3n=20acumul?= =?UTF-8?q?=C3=A1ndola?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 48 +++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 12f25eb6..2939b9f9 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -26,23 +26,43 @@ class UriCollectionJob < PeriodicJob stat = site.stats.create! name: STAT_NAME beginning = beginning_of_interval + # Recorremos todos los hostnames y uris posibles y luego agrupamos + # recursivamente para no tener que recalcular, asumiendo que es más + # rápido buscar en los rollups indexados que en la tabla en bruto. + # + # Los referers solo se agrupan por host. site.hostnames.each do |host| break if stop? + dimensions = { host: host } + uris.each do |uri| break if stop? - rollup('host|uri', beginning, host: host, uri: uri) + dimensions[:uri] = uri - AccessLog.where(host: host, uri: uri).distinct(:http_referer).pluck(:http_referer).each do |http_referer| - rollup('host|uri|referer', beginning, host: host, uri: uri, http_referer: http_referer) - end + rollup('host|uri', beginning, **dimensions) + reduce_rollup('host|uri', beginning, **dimensions) end - rollup('host', beginning, host: host) + dimensions.delete(:uri) + # Reducir todas las visitas a cantidad de visitas por host + recursive_rollup(name: 'host|uri', + new_name: 'host', + interval_previous: starting_interval, + interval: starting_interval, + dimensions: dimensions) + + # Acumular por mes y año + reduce_rollup('host', beginning, **dimensions) + + # Obtener orígenes de visitas por host AccessLog.where(host: host).distinct(:http_referer).pluck(:http_referer).each do |http_referer| - rollup('host|referer', beginning, host: host, http_referer: http_referer) + dimensions[:http_referer] = http_referer + + rollup('host|referer', beginning, **dimensions) + reduce_rollup('host|referer', beginning, **dimensions) end end @@ -53,8 +73,7 @@ class UriCollectionJob < PeriodicJob private - # Generar un rollup a partir de unas dimensiones que también sirven de - # filtro. + # Generar un rollup de access logs # # @param :name [String] # @param :beginning [Time] @@ -67,9 +86,16 @@ class UriCollectionJob < PeriodicJob .non_robots .group(*dimensions.keys) .rollup(name, interval: starting_interval, update: true) + end - # Reducir las estadísticas calculadas aplicando un rollup sobre el - # intervalo más amplio. + # Reducir las estadísticas calculadas aplicando un rollup sobre el + # intervalo más amplio. + # + # @param :name [String] + # @param :beginning [Time] + # @param :dimensions [Hash] + # @return [nil] + def reduce_rollup(name, beginning, **dimensions) Stat::INTERVALS.reduce do |previous, current| recursive_rollup(name: name, interval_previous: previous, @@ -80,6 +106,8 @@ class UriCollectionJob < PeriodicJob # Devolver el intervalo actual current end + + nil end def stat_name From 23260791e44cf747aa388d7cd1a496d9005639d7 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 15:29:27 -0300 Subject: [PATCH 069/121] =?UTF-8?q?fixup!=20recolectar=20informaci=C3=B3n?= =?UTF-8?q?=20acumul=C3=A1ndola?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 2939b9f9..bea388a7 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -52,7 +52,8 @@ class UriCollectionJob < PeriodicJob new_name: 'host', interval_previous: starting_interval, interval: starting_interval, - dimensions: dimensions) + dimensions: dimensions, + beginning: beginning) # Acumular por mes y año reduce_rollup('host', beginning, **dimensions) From cc839affb866b040e84e836a95be2294ab872934 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 17:43:06 -0300 Subject: [PATCH 070/121] tener en cuenta todos los directorios posibles por ejemplo DeployLocalizedDomain en la otra rama tiene otras rutas --- app/jobs/uri_collection_job.rb | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index bea388a7..8504f006 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -115,11 +115,17 @@ class UriCollectionJob < PeriodicJob STAT_NAME end + # Obtiene todas las ubicaciones de archivos + # # @return [String] # # TODO: Cambiar al mergear origin-referer - def destination - @destination ||= site.deploys.find_by(type: 'DeployLocal').destination + def destinations + @destinations ||= site.deploys.map(&:destination).select do |d| + File.directory?(d) + end.map do |d| + File.realpath(d) + end.uniq end # Recolecta todas las URIs menos imágenes @@ -127,14 +133,16 @@ class UriCollectionJob < PeriodicJob # @return [Array] def uris @uris ||= - locales.map do |locale| - uri = "/#{locale}/".squeeze('/') - dir = File.join(destination, locale) + destinations.map do |destination| + locales.map do |locale| + uri = "/#{locale}/".squeeze('/') + dir = File.join(destination, locale) - files(dir).map do |f| - uri + f + files(dir).map do |f| + uri + f + end end - end.flatten(2) + end.flatten(3) end # @return [Array] From 9998f19087318910b70ad69e998bc1c0aaf809a2 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 17:57:29 -0300 Subject: [PATCH 071/121] =?UTF-8?q?acumular=20visitas=20por=20pa=C3=ADs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 8504f006..eaa3b153 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -58,12 +58,20 @@ class UriCollectionJob < PeriodicJob # Acumular por mes y año reduce_rollup('host', beginning, **dimensions) - # Obtener orígenes de visitas por host - AccessLog.where(host: host).distinct(:http_referer).pluck(:http_referer).each do |http_referer| - dimensions[:http_referer] = http_referer + columns = %i[http_referer geoip2_data_country_name] + columns.each do |column| + # Obtener orígenes de visitas por host + AccessLog.where(host: host).distinct(column).pluck(column).each do |value| + dimensions.delete_if do |k, _| + columns.include? k + end - rollup('host|referer', beginning, **dimensions) - reduce_rollup('host|referer', beginning, **dimensions) + name = "host|#{column}" + dimensions[column] = value + + rollup(name, beginning, **dimensions) + reduce_rollup(name, beginning, **dimensions) + end end end From 23e2edf259759ab0b85bddbb57bfa81bc2c0b677 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 18:01:07 -0300 Subject: [PATCH 072/121] =?UTF-8?q?la=20validaci=C3=B3n=20de=20directorio?= =?UTF-8?q?=20pasa=20por=20otro=20lado?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index eaa3b153..0c5b6b1f 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -20,7 +20,6 @@ class UriCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id - return unless File.directory? destination # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME From d56eeb465a8c91f8874a60d5babfe7baad8071ef Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 18:04:43 -0300 Subject: [PATCH 073/121] =?UTF-8?q?comprobar=20que=20los=20idiomas=20tambi?= =?UTF-8?q?=C3=A9n=20existan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 0c5b6b1f..b6d4a21b 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -145,11 +145,13 @@ class UriCollectionJob < PeriodicJob uri = "/#{locale}/".squeeze('/') dir = File.join(destination, locale) + next unless File.directory? dir + files(dir).map do |f| uri + f end end - end.flatten(3) + end.flatten(3).compact end # @return [Array] From 7dad3e9e54f523f7d4b1e801aede6dd4bfb40e5e Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 20:06:07 -0300 Subject: [PATCH 074/121] =?UTF-8?q?ser=20m=C3=A1s=20explicitxs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index b6d4a21b..d8b24e3a 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -33,39 +33,30 @@ class UriCollectionJob < PeriodicJob site.hostnames.each do |host| break if stop? - dimensions = { host: host } - uris.each do |uri| break if stop? - dimensions[:uri] = uri - - rollup('host|uri', beginning, **dimensions) - reduce_rollup('host|uri', beginning, **dimensions) + rollup('host|uri', beginning, host: host, uri: uri) + reduce_rollup('host|uri', beginning, host: host, uri: uri) end - dimensions.delete(:uri) - # Reducir todas las visitas a cantidad de visitas por host recursive_rollup(name: 'host|uri', new_name: 'host', interval_previous: starting_interval, interval: starting_interval, - dimensions: dimensions, + dimensions: { host: host }, beginning: beginning) # Acumular por mes y año - reduce_rollup('host', beginning, **dimensions) + reduce_rollup('host', beginning, host: host) columns = %i[http_referer geoip2_data_country_name] columns.each do |column| # Obtener orígenes de visitas por host AccessLog.where(host: host).distinct(column).pluck(column).each do |value| - dimensions.delete_if do |k, _| - columns.include? k - end - name = "host|#{column}" + dimensions = { host: host } dimensions[column] = value rollup(name, beginning, **dimensions) From 595a8759a4379ea2e7bad904afe3bb46142207a7 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 29 Apr 2022 20:32:21 -0300 Subject: [PATCH 075/121] usar las uris como fuente de verdad --- app/jobs/uri_collection_job.rb | 45 ++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index d8b24e3a..310984a0 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -24,6 +24,7 @@ class UriCollectionJob < PeriodicJob # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME beginning = beginning_of_interval + columns = %i[http_referer geoip2_data_country_name] # Recorremos todos los hostnames y uris posibles y luego agrupamos # recursivamente para no tener que recalcular, asumiendo que es más @@ -33,11 +34,31 @@ class UriCollectionJob < PeriodicJob site.hostnames.each do |host| break if stop? + host_dimensions = { host: host } + + # Las URIs son la fuente de verdad de las visitas, porque son las + # que indican las páginas y recursos descargables, el resto son + # imágenes, CSS, JS y tipografías que no nos aportan números + # significativos. uris.each do |uri| break if stop? - rollup('host|uri', beginning, host: host, uri: uri) - reduce_rollup('host|uri', beginning, host: host, uri: uri) + name = 'host|uri' + dimensions = { host: host, uri: uri } + + rollup(name, beginning, **dimensions) + reduce_rollup(name, beginning, **dimensions) + + columns.each do |column| + # Obtener orígenes de visitas por host + AccessLog.where(**host_dimensions).distinct(column).pluck(column).each do |value| + name = "#{name}|#{column}" + dimensions[column] = value + + rollup(name, beginning, **dimensions) + reduce_rollup(name, beginning, **dimensions) + end + end end # Reducir todas las visitas a cantidad de visitas por host @@ -45,23 +66,21 @@ class UriCollectionJob < PeriodicJob new_name: 'host', interval_previous: starting_interval, interval: starting_interval, - dimensions: { host: host }, + dimensions: host_dimensions, beginning: beginning) # Acumular por mes y año - reduce_rollup('host', beginning, host: host) + reduce_rollup('host', beginning, **host_dimensions) - columns = %i[http_referer geoip2_data_country_name] columns.each do |column| - # Obtener orígenes de visitas por host - AccessLog.where(host: host).distinct(column).pluck(column).each do |value| - name = "host|#{column}" - dimensions = { host: host } - dimensions[column] = value + recursive_rollup(name: "host|uri|#{column}", + new_name: "host|#{column}", + interval_previous: starting_interval, + interval: starting_interval, + dimensions: host_dimensions, + beginning: beginning) - rollup(name, beginning, **dimensions) - reduce_rollup(name, beginning, **dimensions) - end + reduce_rollup("host|#{column}", beginning, **host_dimensions) end end From 121d4f49e26505f2f4edb2ef095a6ead487d83ab Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 09:56:19 -0300 Subject: [PATCH 076/121] no se puede renombrar un rollup usando dimensiones exactas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit además era un método que hacía dos cosas distintas según un parámetro --- app/jobs/concerns/recursive_rollup.rb | 27 +++++++++++++++++++++++---- app/jobs/uri_collection_job.rb | 22 ++++++++++------------ 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index b3327a73..5ba9dd8b 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -15,14 +15,33 @@ module RecursiveRollup # @param :interval [String] # @param :operation [Symbol] # @param :dimensions [Hash] - # @param :new_name [String] # @param :beginning [Time] # @return [Rollup] - def recursive_rollup(name:, interval_previous:, interval:, dimensions:, beginning:, operation: :sum, new_name: nil) - Rollup.where(name: name, interval: interval_previous, dimensions: dimensions.to_json) + def recursive_rollup(name:, interval_previous:, interval:, dimensions:, beginning:, operation: :sum) + Rollup.where(name: name, interval: interval_previous, dimensions: dimensions) .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) .group(*dimensions_to_jsonb_query(dimensions)) - .rollup(new_name || name, interval: interval, update: true) do |rollup| + .rollup(name, interval: interval, update: true) do |rollup| + rollup.try(operation, :value) + end + end + + # Genera un nuevo rollup a partir de uno anterior. + # + # @param :name [String] + # @param :new_name [String] + # @param :interval_previous [String] + # @param :interval [String] + # @param :operation [Symbol] + # @param :dimensions [Hash] + # @param :beginning [Time] + # @return [Rollup] + def square_rollup(name:, new_name:, interval:, dimensions:, beginning:, operation: :sum) + Rollup.where(name: name, interval: interval_previous) + .where_dimensions(**dimensions) + .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) + .group(*dimensions_to_jsonb_query(dimensions)) + .rollup(new_name, interval: interval, update: true) do |rollup| rollup.try(operation, :value) end end diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 310984a0..5de95d43 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -62,23 +62,21 @@ class UriCollectionJob < PeriodicJob end # Reducir todas las visitas a cantidad de visitas por host - recursive_rollup(name: 'host|uri', - new_name: 'host', - interval_previous: starting_interval, - interval: starting_interval, - dimensions: host_dimensions, - beginning: beginning) + square_rollup(name: 'host|uri', + new_name: 'host', + interval: starting_interval, + dimensions: host_dimensions, + beginning: beginning) # Acumular por mes y año reduce_rollup('host', beginning, **host_dimensions) columns.each do |column| - recursive_rollup(name: "host|uri|#{column}", - new_name: "host|#{column}", - interval_previous: starting_interval, - interval: starting_interval, - dimensions: host_dimensions, - beginning: beginning) + square_rollup(name: "host|uri|#{column}", + new_name: "host|#{column}", + interval: starting_interval, + dimensions: host_dimensions, + beginning: beginning) reduce_rollup("host|#{column}", beginning, **host_dimensions) end From b36bdc976324d7359c8ac2a0574ff6e949fdb559 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 10:03:14 -0300 Subject: [PATCH 077/121] fixup! no se puede renombrar un rollup usando dimensiones exactas --- app/jobs/concerns/recursive_rollup.rb | 6 +++--- app/jobs/uri_collection_job.rb | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index 5ba9dd8b..3539d7d2 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -30,14 +30,14 @@ module RecursiveRollup # # @param :name [String] # @param :new_name [String] - # @param :interval_previous [String] - # @param :interval [String] + # @param :interval_previous [String, Symbol] + # @param :interval [String, Symbol] # @param :operation [Symbol] # @param :dimensions [Hash] # @param :beginning [Time] # @return [Rollup] def square_rollup(name:, new_name:, interval:, dimensions:, beginning:, operation: :sum) - Rollup.where(name: name, interval: interval_previous) + Rollup.where(name: name, interval: interval) .where_dimensions(**dimensions) .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) .group(*dimensions_to_jsonb_query(dimensions)) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 5de95d43..a9549efe 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -52,11 +52,11 @@ class UriCollectionJob < PeriodicJob columns.each do |column| # Obtener orígenes de visitas por host AccessLog.where(**host_dimensions).distinct(column).pluck(column).each do |value| - name = "#{name}|#{column}" + column_name = "host|#{column}" dimensions[column] = value - rollup(name, beginning, **dimensions) - reduce_rollup(name, beginning, **dimensions) + rollup(column_name, beginning, **dimensions) + reduce_rollup(column_name, beginning, **dimensions) end end end @@ -145,6 +145,9 @@ class UriCollectionJob < PeriodicJob # Recolecta todas las URIs menos imágenes # + # TODO: Para los sitios con DeployLocalizedDomain estamos buscando + # URIs de más. + # # @return [Array] def uris @uris ||= From ce0d2306f6f8b14ea046c77d41ff9a106559bae4 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 10:13:49 -0300 Subject: [PATCH 078/121] fixup! fixup! no se puede renombrar un rollup usando dimensiones exactas --- app/jobs/uri_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index a9549efe..1f635997 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -52,7 +52,7 @@ class UriCollectionJob < PeriodicJob columns.each do |column| # Obtener orígenes de visitas por host AccessLog.where(**host_dimensions).distinct(column).pluck(column).each do |value| - column_name = "host|#{column}" + column_name = "host|uri|#{column}" dimensions[column] = value rollup(column_name, beginning, **dimensions) From deb3f0f0709bed4d586e8580cb1987722b310094 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 10:23:58 -0300 Subject: [PATCH 079/121] =?UTF-8?q?usar=20el=20=C3=BAltimo=20intervalo,=20?= =?UTF-8?q?no=20el=20que=20acabamos=20de=20crear?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 1f635997..e5105e5a 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -21,9 +21,10 @@ class UriCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id + # Obtener el principio del intervalo anterior + beginning = beginning_of_interval # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME - beginning = beginning_of_interval columns = %i[http_referer geoip2_data_country_name] # Recorremos todos los hostnames y uris posibles y luego agrupamos From 80f2df362b3091b8c9856766ceb1bf4bd5dc4f64 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 10:24:21 -0300 Subject: [PATCH 080/121] precalcular las columnas --- app/jobs/uri_collection_job.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index e5105e5a..196d2e50 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -25,7 +25,8 @@ class UriCollectionJob < PeriodicJob beginning = beginning_of_interval # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME - columns = %i[http_referer geoip2_data_country_name] + # Columnas a agrupar + columns = %i[http_referer geoip2_data_country_name].zip([nil]).to_h # Recorremos todos los hostnames y uris posibles y luego agrupamos # recursivamente para no tener que recalcular, asumiendo que es más @@ -36,6 +37,9 @@ class UriCollectionJob < PeriodicJob break if stop? host_dimensions = { host: host } + columns.each_key do |column| + columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) + end # Las URIs son la fuente de verdad de las visitas, porque son las # que indican las páginas y recursos descargables, el resto son @@ -50,9 +54,9 @@ class UriCollectionJob < PeriodicJob rollup(name, beginning, **dimensions) reduce_rollup(name, beginning, **dimensions) - columns.each do |column| + columns.each_pair do |column, values| # Obtener orígenes de visitas por host - AccessLog.where(**host_dimensions).distinct(column).pluck(column).each do |value| + values.each do |value| column_name = "host|uri|#{column}" dimensions[column] = value @@ -72,7 +76,7 @@ class UriCollectionJob < PeriodicJob # Acumular por mes y año reduce_rollup('host', beginning, **host_dimensions) - columns.each do |column| + columns.each_key do |column| square_rollup(name: "host|uri|#{column}", new_name: "host|#{column}", interval: starting_interval, From e9d13a7c5931b9ad4e86bf218ff6aff79ac132c0 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 10:49:32 -0300 Subject: [PATCH 081/121] mostrar demora --- app/jobs/uri_collection_job.rb | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 196d2e50..ae4b098e 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -38,7 +38,10 @@ class UriCollectionJob < PeriodicJob host_dimensions = { host: host } columns.each_key do |column| + time_start = Time.now columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) + + puts "#{column}: #{Time.now - time_start}" end # Las URIs son la fuente de verdad de las visitas, porque son las @@ -51,39 +54,58 @@ class UriCollectionJob < PeriodicJob name = 'host|uri' dimensions = { host: host, uri: uri } + time_start = Time.now rollup(name, beginning, **dimensions) + puts "#{dimensions.values.join}: #{Time.now - time_start}" + + time_start = Time.now reduce_rollup(name, beginning, **dimensions) + puts "reduced: #{Time.now - time_start}" columns.each_pair do |column, values| # Obtener orígenes de visitas por host values.each do |value| + time_start = Time.now + column_name = "host|uri|#{column}" dimensions[column] = value rollup(column_name, beginning, **dimensions) + puts "#{dimensions.values.join}: #{Time.now - time_start}" + + time_start = Time.now reduce_rollup(column_name, beginning, **dimensions) + puts "reduced: #{Time.now - time_start}" end end end # Reducir todas las visitas a cantidad de visitas por host + time_start = Time.now square_rollup(name: 'host|uri', new_name: 'host', interval: starting_interval, dimensions: host_dimensions, beginning: beginning) + puts "#{host}: #{Time.now - time_start}" # Acumular por mes y año + time_start = Time.now reduce_rollup('host', beginning, **host_dimensions) + puts "reduced: #{Time.now - time_start}" columns.each_key do |column| + time_start = Time.now square_rollup(name: "host|uri|#{column}", new_name: "host|#{column}", interval: starting_interval, dimensions: host_dimensions, beginning: beginning) + puts "#{host} #{column}: #{Time.now - time_start}" + time_start = Time.now reduce_rollup("host|#{column}", beginning, **host_dimensions) + puts "reduced: #{Time.now - time_start}" end end From 1962175cc66502848aa0e17116959c3e0986f140 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 10:58:41 -0300 Subject: [PATCH 082/121] Revert "mostrar demora" This reverts commit e9d13a7c5931b9ad4e86bf218ff6aff79ac132c0. --- app/jobs/uri_collection_job.rb | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index ae4b098e..196d2e50 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -38,10 +38,7 @@ class UriCollectionJob < PeriodicJob host_dimensions = { host: host } columns.each_key do |column| - time_start = Time.now columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) - - puts "#{column}: #{Time.now - time_start}" end # Las URIs son la fuente de verdad de las visitas, porque son las @@ -54,58 +51,39 @@ class UriCollectionJob < PeriodicJob name = 'host|uri' dimensions = { host: host, uri: uri } - time_start = Time.now rollup(name, beginning, **dimensions) - puts "#{dimensions.values.join}: #{Time.now - time_start}" - - time_start = Time.now reduce_rollup(name, beginning, **dimensions) - puts "reduced: #{Time.now - time_start}" columns.each_pair do |column, values| # Obtener orígenes de visitas por host values.each do |value| - time_start = Time.now - column_name = "host|uri|#{column}" dimensions[column] = value rollup(column_name, beginning, **dimensions) - puts "#{dimensions.values.join}: #{Time.now - time_start}" - - time_start = Time.now reduce_rollup(column_name, beginning, **dimensions) - puts "reduced: #{Time.now - time_start}" end end end # Reducir todas las visitas a cantidad de visitas por host - time_start = Time.now square_rollup(name: 'host|uri', new_name: 'host', interval: starting_interval, dimensions: host_dimensions, beginning: beginning) - puts "#{host}: #{Time.now - time_start}" # Acumular por mes y año - time_start = Time.now reduce_rollup('host', beginning, **host_dimensions) - puts "reduced: #{Time.now - time_start}" columns.each_key do |column| - time_start = Time.now square_rollup(name: "host|uri|#{column}", new_name: "host|#{column}", interval: starting_interval, dimensions: host_dimensions, beginning: beginning) - puts "#{host} #{column}: #{Time.now - time_start}" - time_start = Time.now reduce_rollup("host|#{column}", beginning, **host_dimensions) - puts "reduced: #{Time.now - time_start}" end end From 6699db838078d141cc3eaab7815b09d22d1862ec Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 11:34:21 -0300 Subject: [PATCH 083/121] otra vez el mismo error de hashes acumulados hice benchmarks, generar hashes nuevos es mas rapido que duplicarlos, y eliminar las keys que no corresponden es apenas mas rapido pero con mas posibilidad de bugs --- app/jobs/uri_collection_job.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 196d2e50..620123db 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -58,10 +58,11 @@ class UriCollectionJob < PeriodicJob # Obtener orígenes de visitas por host values.each do |value| column_name = "host|uri|#{column}" - dimensions[column] = value + column_dimensions = { host: host, uri: uri } + column_dimensions[column] = value - rollup(column_name, beginning, **dimensions) - reduce_rollup(column_name, beginning, **dimensions) + rollup(column_name, beginning, **column_dimensions) + reduce_rollup(column_name, beginning, **column_dimensions) end end end From 6bbb5b48c0c6f63fa3ee2601affcdd51d94cb79e Mon Sep 17 00:00:00 2001 From: Nulo Date: Sat, 30 Apr 2022 14:56:04 +0000 Subject: [PATCH 084/121] editor: Borrar comentario viejo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ahora la blockquote está implementada por https://0xacab.org/sutty/sutty/-/merge_requests/83, está implementada como bloque normal ya que una blockquote puede tener varios blockquotes adentro --- app/javascript/editor/types/parentBlocks.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/app/javascript/editor/types/parentBlocks.ts b/app/javascript/editor/types/parentBlocks.ts index ffe40bdf..5ba8f366 100644 --- a/app/javascript/editor/types/parentBlocks.ts +++ b/app/javascript/editor/types/parentBlocks.ts @@ -20,7 +20,6 @@ function makeParentBlock( }; } -// TODO: añadir blockquote // XXX: si agregás algo acá, probablemente le quieras hacer un botón // en app/views/posts/attributes/_content.haml export const parentBlocks: { [propName: string]: EditorNode } = { From 001443c1be55fc0f7f893392fd9e119a1c07c125 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 14:07:34 -0300 Subject: [PATCH 085/121] =?UTF-8?q?la=20=C3=BAltima=20actualizaci=C3=B3n?= =?UTF-8?q?=20es=20cuando=20se=20cierra=20el=20stat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/stats/index.haml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 2b5d643e..02683781 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -6,8 +6,8 @@ %p %small = t('.last_update') - %time{ datetime: @last_stat.created_at } - #{time_ago_in_words @last_stat.created_at}. + %time{ datetime: @last_stat.updated_at } + #{time_ago_in_words @last_stat.updated_at}. .mb-5 - Stat::INTERVALS.each do |interval| From 6bf286077bba19858665314f9fd850133229c7eb Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 14:39:31 -0300 Subject: [PATCH 086/121] =?UTF-8?q?generar=20estad=C3=ADsticas=20de=20uso?= =?UTF-8?q?=20de=20infra?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/concerns/recursive_rollup.rb | 24 +++++++++++++++++++++ app/jobs/stat_collection_job.rb | 22 +++++--------------- app/jobs/uri_collection_job.rb | 30 ++++----------------------- 3 files changed, 33 insertions(+), 43 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index 3539d7d2..28c2e66a 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -46,6 +46,30 @@ module RecursiveRollup end end + # Reducir las estadísticas calculadas aplicando un rollup sobre el + # intervalo más amplio. + # + # @param :name [String] + # @param :beginning [Time] + # @param :operation [Symbol] + # @param :dimensions [Hash] + # @return [nil] + def reduce_rollup(name, beginning, operation, **dimensions) + Stat::INTERVALS.reduce do |previous, current| + recursive_rollup(name: name, + interval_previous: previous, + interval: current, + dimensions: dimensions, + beginning: beginning, + operation: operation) + + # Devolver el intervalo actual + current + end + + nil + end + # @param :dimensions [Hash] # @return [Array] def dimensions_to_jsonb_query(dimensions) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index a67a3c50..65226b5c 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -9,10 +9,8 @@ class StatCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id - - # Registrar que se hicieron todas las recolecciones - stat = site.stats.create! name: STAT_NAME beginning = beginning_of_interval + stat = site.stats.create! name: STAT_NAME scope.rollup('builds', **options) @@ -24,16 +22,9 @@ class StatCollectionJob < PeriodicJob rollup.average(:seconds) end - # XXX: Es correcto promediar promedios? - Stat::INTERVALS.reduce do |previous, current| - opts = { interval_previous: previous, interval: current, beginning: beginning, dimensions: { site_id: site.id } } - - recursive_rollup(name: 'builds', **opts) - recursive_rollup(name: 'space_used', operation: :average, **opts) - recursive_rollup(name: 'build_time', operation: :average, **opts) - - current - end + reduce_rollup('builds', beginning, :sum, site_id: site_id) + reduce_rollup('space_used', beginning, :average, site_id: site_id) + reduce_rollup('build_time', beginning, :average, site_id: site_id) stat.touch run_again! unless once @@ -45,10 +36,7 @@ class StatCollectionJob < PeriodicJob # # @return [ActiveRecord::Relation] def scope - @scope ||= site.build_stats - .jekyll - .where('created_at >= ?', beginning_of_interval) - .group(:site_id) + @scope ||= site.build_stats.jekyll.where('created_at >= ?', beginning_of_interval) end # Las opciones por defecto diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 620123db..1553fb22 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -52,7 +52,7 @@ class UriCollectionJob < PeriodicJob dimensions = { host: host, uri: uri } rollup(name, beginning, **dimensions) - reduce_rollup(name, beginning, **dimensions) + reduce_rollup(name, beginning, :sum, **dimensions) columns.each_pair do |column, values| # Obtener orígenes de visitas por host @@ -62,7 +62,7 @@ class UriCollectionJob < PeriodicJob column_dimensions[column] = value rollup(column_name, beginning, **column_dimensions) - reduce_rollup(column_name, beginning, **column_dimensions) + reduce_rollup(column_name, beginning, :sum, **column_dimensions) end end end @@ -75,7 +75,7 @@ class UriCollectionJob < PeriodicJob beginning: beginning) # Acumular por mes y año - reduce_rollup('host', beginning, **host_dimensions) + reduce_rollup('host', beginning, :sum, **host_dimensions) columns.each_key do |column| square_rollup(name: "host|uri|#{column}", @@ -84,7 +84,7 @@ class UriCollectionJob < PeriodicJob dimensions: host_dimensions, beginning: beginning) - reduce_rollup("host|#{column}", beginning, **host_dimensions) + reduce_rollup("host|#{column}", beginning, :sum, **host_dimensions) end end @@ -110,28 +110,6 @@ class UriCollectionJob < PeriodicJob .rollup(name, interval: starting_interval, update: true) end - # Reducir las estadísticas calculadas aplicando un rollup sobre el - # intervalo más amplio. - # - # @param :name [String] - # @param :beginning [Time] - # @param :dimensions [Hash] - # @return [nil] - def reduce_rollup(name, beginning, **dimensions) - Stat::INTERVALS.reduce do |previous, current| - recursive_rollup(name: name, - interval_previous: previous, - interval: current, - dimensions: dimensions, - beginning: beginning) - - # Devolver el intervalo actual - current - end - - nil - end - def stat_name STAT_NAME end From 8d4958af536a6e9a0dedcd9381811e5898fee91f Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 14:44:27 -0300 Subject: [PATCH 087/121] =?UTF-8?q?fixup!=20generar=20estad=C3=ADsticas=20?= =?UTF-8?q?de=20uso=20de=20infra?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 65226b5c..3f7cdd48 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -36,7 +36,7 @@ class StatCollectionJob < PeriodicJob # # @return [ActiveRecord::Relation] def scope - @scope ||= site.build_stats.jekyll.where('created_at >= ?', beginning_of_interval) + @scope ||= site.build_stats.jekyll.where('created_at >= ?', beginning_of_interval).group(:site_id) end # Las opciones por defecto From 92cd2ac05236c206d69cf081bec8d9e3fd658dc2 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 14:48:42 -0300 Subject: [PATCH 088/121] =?UTF-8?q?fixup!=20fixup!=20generar=20estad=C3=AD?= =?UTF-8?q?sticas=20de=20uso=20de=20infra?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 3f7cdd48..63d44999 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -36,7 +36,7 @@ class StatCollectionJob < PeriodicJob # # @return [ActiveRecord::Relation] def scope - @scope ||= site.build_stats.jekyll.where('created_at >= ?', beginning_of_interval).group(:site_id) + @scope ||= site.build_stats.jekyll.where('build_stats.created_at >= ?', beginning_of_interval).group(:site_id) end # Las opciones por defecto From 756fb8f1cbc32561bfd1433b0698267024d870ff Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 16:38:16 -0300 Subject: [PATCH 089/121] =?UTF-8?q?usar=20el=20id=20del=20sitio=20como=20d?= =?UTF-8?q?imensi=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 9a8a345d..3e1ec76b 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -60,12 +60,7 @@ class StatsController < ApplicationController def resources return unless stale? [last_stat, interval, resource, period] - options = { - interval: interval, - dimensions: { - deploy_id: @site.deploys.where(type: 'DeployLocal').pluck(:id).first - } - } + options = { interval: interval, dimensions: { site_id: site.id } } render json: rollup_scope.series(resource, **options) end From d181c179ff48948143526f4e63c6ac8b9bc40d75 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 16:38:32 -0300 Subject: [PATCH 090/121] nota --- app/controllers/stats_controller.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 3e1ec76b..35f7f108 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -69,6 +69,8 @@ class StatsController < ApplicationController return unless stale? [last_stat, hostnames, interval, normalized_urls, period] options = { host: hostnames, uri: normalized_paths } + # XXX: where_dimensions es más corto pero no aprovecha los índices + # de Rollup stats = rollup_scope.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series| series.each do |serie| serie[:name] = serie[:dimensions].slice('host', 'uri').values.join.sub('/index.html', '/') From 604ee233bd1ef7bbbfa5ccd293a7d2edd6ca248e Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 16:38:50 -0300 Subject: [PATCH 091/121] =?UTF-8?q?agregar=20la=20dimensi=C3=B3n=20al=20ag?= =?UTF-8?q?rupar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 1553fb22..44642c1f 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -78,13 +78,18 @@ class UriCollectionJob < PeriodicJob reduce_rollup('host', beginning, :sum, **host_dimensions) columns.each_key do |column| - square_rollup(name: "host|uri|#{column}", - new_name: "host|#{column}", - interval: starting_interval, - dimensions: host_dimensions, - beginning: beginning) + values.each do |value| + column_dimensions = { host: host } + column_dimensions[column] = value - reduce_rollup("host|#{column}", beginning, :sum, **host_dimensions) + square_rollup(name: "host|uri|#{column}", + new_name: "host|#{column}", + interval: starting_interval, + dimensions: column_dimensions, + beginning: beginning) + + reduce_rollup("host|#{column}", beginning, :sum, **column_dimensions) + end end end From d766ee4eff68284e28462f1afe3e9c6c6d061698 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 16:44:23 -0300 Subject: [PATCH 092/121] =?UTF-8?q?fixup!=20agregar=20la=20dimensi=C3=B3n?= =?UTF-8?q?=20al=20agrupar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 44642c1f..a0b1181e 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -77,7 +77,7 @@ class UriCollectionJob < PeriodicJob # Acumular por mes y año reduce_rollup('host', beginning, :sum, **host_dimensions) - columns.each_key do |column| + columns.each_pair do |column, values| values.each do |value| column_dimensions = { host: host } column_dimensions[column] = value From fb34823ce31606c23823d66365ef85438e0e12fa Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 17:51:05 -0300 Subject: [PATCH 093/121] deprecar square_rollup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit es muy lento buscar dentro de cada dimensión --- app/jobs/concerns/recursive_rollup.rb | 20 -------------------- app/jobs/uri_collection_job.rb | 13 ++----------- 2 files changed, 2 insertions(+), 31 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index 28c2e66a..4d9e11b0 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -26,26 +26,6 @@ module RecursiveRollup end end - # Genera un nuevo rollup a partir de uno anterior. - # - # @param :name [String] - # @param :new_name [String] - # @param :interval_previous [String, Symbol] - # @param :interval [String, Symbol] - # @param :operation [Symbol] - # @param :dimensions [Hash] - # @param :beginning [Time] - # @return [Rollup] - def square_rollup(name:, new_name:, interval:, dimensions:, beginning:, operation: :sum) - Rollup.where(name: name, interval: interval) - .where_dimensions(**dimensions) - .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) - .group(*dimensions_to_jsonb_query(dimensions)) - .rollup(new_name, interval: interval, update: true) do |rollup| - rollup.try(operation, :value) - end - end - # Reducir las estadísticas calculadas aplicando un rollup sobre el # intervalo más amplio. # diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index a0b1181e..fe50929d 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -68,11 +68,7 @@ class UriCollectionJob < PeriodicJob end # Reducir todas las visitas a cantidad de visitas por host - square_rollup(name: 'host|uri', - new_name: 'host', - interval: starting_interval, - dimensions: host_dimensions, - beginning: beginning) + rollup('host', beginning, :sum, **host_dimensions) # Acumular por mes y año reduce_rollup('host', beginning, :sum, **host_dimensions) @@ -82,12 +78,7 @@ class UriCollectionJob < PeriodicJob column_dimensions = { host: host } column_dimensions[column] = value - square_rollup(name: "host|uri|#{column}", - new_name: "host|#{column}", - interval: starting_interval, - dimensions: column_dimensions, - beginning: beginning) - + rollup("host|#{column}", beginning, :sum, **column_dimensions) reduce_rollup("host|#{column}", beginning, :sum, **column_dimensions) end end From e00780c7f0d7fee3e0364ffb1fc7cfc7ac5b237c Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 17:55:53 -0300 Subject: [PATCH 094/121] mostrar la cantidad de visitas por origen y pais --- app/controllers/stats_controller.rb | 14 ++++++++++++++ app/jobs/uri_collection_job.rb | 2 +- app/models/stat.rb | 8 ++++++++ app/views/stats/index.haml | 21 +++++++++++++++++++++ config/locales/en.yml | 11 +++++++++++ config/locales/es.yml | 11 +++++++++++ 6 files changed, 66 insertions(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 35f7f108..84ca3028 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -39,6 +39,20 @@ class StatsController < ApplicationController last_stat chart_options normalized_urls + + expires_in = Time.now.try(:"end_of_#{Stat.default_interval}") - Time.now + @columns = {} + + Stat::COLUMNS.each do |column| + @columns[column] = + Rails.cache.fetch("stats/#{column}/#{site.id}", expires_in: expires_in) do + Rollup.where(interval: interval, name: "host|#{column}") + .where_dimensions(host: hostnames) + .group("dimensions->>'#{column}'") + .sum(:value) + .transform_values(&:to_i) + end + end end # Genera un gráfico de visitas por dominio asociado a este sitio diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index fe50929d..a7dd2a00 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -26,7 +26,7 @@ class UriCollectionJob < PeriodicJob # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME # Columnas a agrupar - columns = %i[http_referer geoip2_data_country_name].zip([nil]).to_h + columns = Stat::COLUMNS.zip([nil]).to_h # Recorremos todos los hostnames y uris posibles y luego agrupamos # recursivamente para no tener que recalcular, asumiendo que es más diff --git a/app/models/stat.rb b/app/models/stat.rb index 5f72ccd0..f1016026 100644 --- a/app/models/stat.rb +++ b/app/models/stat.rb @@ -5,6 +5,14 @@ class Stat < ApplicationRecord # XXX: Los intervalos van en orden de mayor especificidad a menor INTERVALS = %i[day month year].freeze RESOURCES = %i[builds space_used build_time].freeze + COLUMNS = %i[http_referer geoip2_data_country_name].freeze belongs_to :site + + # El intervalo por defecto + # + # @return [Symbol] + def self.default_interval + INTERVALS.first + end end diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 02683781..be0c1426 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -32,6 +32,27 @@ - if @normalized_urls.present? = line_chart site_stats_uris_path(urls: @normalized_urls, **@chart_params), **@chart_options + .row.mb-5.row-cols-1.row-cols-md-2 + - @columns.each_pair do |column, values| + - next if values.blank? + .col.mb-5 + %h2= t(".columns.#{column}.title") + %p.lead= t(".columns.#{column}.description") + + %table.table + %colgroup + %col + %col + %thead + %tr.sticky-top.background-white + %th{ scope: 'col' }= t(".columns.#{column}.column") + %th{ scope: 'col' }= t('.columns.visits') + %tfoot + %tbody + - values.each_pair do |col, val| + %tr + %th{ scope: 'row', style: 'word-break: break-all' }= col.blank? ? t('.columns.empty') : col + %td= val .mb-5 %h2= t('.resources.title') %p.lead= t('.resources.description') diff --git a/config/locales/en.yml b/config/locales/en.yml index 732eb4bd..76760958 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -284,6 +284,17 @@ en: build_time: title: 'Publication time' description: 'Average time your site takes to build.' + columns: + visits: "Visits" + empty: "(nothing)" + http_referer: + title: "Referers" + description: "Visits by origin" + column: "Referer" + geoip2_data_country_name: + title: "Countries" + description: "Visits by country" + column: "Country" sites: donations: url: 'https://donaciones.sutty.nl/en/' diff --git a/config/locales/es.yml b/config/locales/es.yml index e47c1fb5..82bfd253 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -289,6 +289,17 @@ es: build_time: title: 'Tiempo de publicación' description: 'Tiempo promedio que toma en publicarse tu sitio.' + columns: + visits: "Visitas" + empty: "(nada)" + http_referer: + title: "Referencias" + description: "Orígenes de las visitas" + column: "Referencia" + geoip2_data_country_name: + title: "Países" + description: "Cantidad de visitas por país" + column: "País" sites: donations: url: 'https://donaciones.sutty.nl/' From 224e599de2f9b5acf1ae75ebfc5463b37364d179 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 18:05:08 -0300 Subject: [PATCH 095/121] fixup! mostrar la cantidad de visitas por origen y pais --- app/jobs/uri_collection_job.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index a7dd2a00..c3e48ecd 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -68,7 +68,7 @@ class UriCollectionJob < PeriodicJob end # Reducir todas las visitas a cantidad de visitas por host - rollup('host', beginning, :sum, **host_dimensions) + rollup('host', beginning, **host_dimensions) # Acumular por mes y año reduce_rollup('host', beginning, :sum, **host_dimensions) @@ -78,7 +78,7 @@ class UriCollectionJob < PeriodicJob column_dimensions = { host: host } column_dimensions[column] = value - rollup("host|#{column}", beginning, :sum, **column_dimensions) + rollup("host|#{column}", beginning, **column_dimensions) reduce_rollup("host|#{column}", beginning, :sum, **column_dimensions) end end From 99e3f093fcd1fda9447ebcfefd6661e6053f76a4 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 18:20:45 -0300 Subject: [PATCH 096/121] ordenar de menor a mayor --- app/controllers/stats_controller.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 84ca3028..e00b9822 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -51,6 +51,9 @@ class StatsController < ApplicationController .group("dimensions->>'#{column}'") .sum(:value) .transform_values(&:to_i) + .sort_by do |_, v| + v.last + end.reverse.to_h end end end From 09696529411acdbf190ad45ce7fd3fbe11f7cd31 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 18:22:35 -0300 Subject: [PATCH 097/121] acomodar el valor a la cantidad de nodos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit en realidad para estas visitas no tiene mucho sentido y de hecho es en las que estaríamos perdiendo información, porque nada indica que no hubo otros países y referencias que quedaron en otros nodos... --- app/controllers/stats_controller.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index e00b9822..07755c73 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -51,6 +51,9 @@ class StatsController < ApplicationController .group("dimensions->>'#{column}'") .sum(:value) .transform_values(&:to_i) + .transform_values do |v| + v * nodes + end .sort_by do |_, v| v.last end.reverse.to_h From 8244532c5a0fa5e00a3501b376c5d0d4fe329af7 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 18:30:24 -0300 Subject: [PATCH 098/121] fixup! ordenar de menor a mayor --- app/controllers/stats_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 07755c73..36b72721 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -55,7 +55,7 @@ class StatsController < ApplicationController v * nodes end .sort_by do |_, v| - v.last + v end.reverse.to_h end end From e4009e59dcf73bf84dd9e6f340618e49799ee60d Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 18:42:50 -0300 Subject: [PATCH 099/121] las tablas con cabeceras sticky usan el color de fondo del theme --- app/assets/stylesheets/application.scss | 1 + app/views/posts/index.haml | 4 ++-- app/views/stats/index.haml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/assets/stylesheets/application.scss b/app/assets/stylesheets/application.scss index e806a032..b756759a 100644 --- a/app/assets/stylesheets/application.scss +++ b/app/assets/stylesheets/application.scss @@ -126,6 +126,7 @@ ol.breadcrumb { color: var(--foreground); } +.table tr.sticky-top, .form-control, .custom-file-label { background-color: var(--background); diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 90d65670..accfdb34 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -71,8 +71,8 @@ %table.table{ data: { controller: 'reorder' } } %caption.sr-only= t('posts.caption') %thead - %tr - %th.border-0.background-white.position-sticky{ style: 'top: 0; z-index: 2', colspan: '4' } + %tr.sticky-top + %th.border-0{ colspan: '4' } .d-flex.flex-row.justify-content-between %div = submit_tag t('posts.reorder.submit'), class: 'btn' diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index be0c1426..6bbf596e 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -44,7 +44,7 @@ %col %col %thead - %tr.sticky-top.background-white + %tr.sticky-top %th{ scope: 'col' }= t(".columns.#{column}.column") %th{ scope: 'col' }= t('.columns.visits') %tfoot From e8300364d1e49e1b5877512f8695fda955e44da3 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 19:11:57 -0300 Subject: [PATCH 100/121] pasar todas las uris a la base de datos --- app/jobs/uri_collection_job.rb | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index c3e48ecd..90789aba 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -45,25 +45,21 @@ class UriCollectionJob < PeriodicJob # que indican las páginas y recursos descargables, el resto son # imágenes, CSS, JS y tipografías que no nos aportan números # significativos. - uris.each do |uri| - break if stop? + name = 'host|uri' + dimensions = { host: host, uri: uris } - name = 'host|uri' - dimensions = { host: host, uri: uri } + rollup(name, beginning, **dimensions) + reduce_rollup(name, beginning, :sum, **dimensions) - rollup(name, beginning, **dimensions) - reduce_rollup(name, beginning, :sum, **dimensions) + columns.each_pair do |column, values| + # Obtener orígenes de visitas por host + values.each do |value| + column_name = "host|uri|#{column}" + column_dimensions = { host: host, uri: uris } + column_dimensions[column] = value - columns.each_pair do |column, values| - # Obtener orígenes de visitas por host - values.each do |value| - column_name = "host|uri|#{column}" - column_dimensions = { host: host, uri: uri } - column_dimensions[column] = value - - rollup(column_name, beginning, **column_dimensions) - reduce_rollup(column_name, beginning, :sum, **column_dimensions) - end + rollup(column_name, beginning, **column_dimensions) + reduce_rollup(column_name, beginning, :sum, **column_dimensions) end end From 26b2d34b4d031dd21a31c490ecd11bb5b3cb8064 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 19:35:14 -0300 Subject: [PATCH 101/121] =?UTF-8?q?simplificar=20el=20c=C3=B3digo!!?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit envía todos los parámetros a la base de datos para acelerar por un montón la generación de datos --- app/jobs/concerns/recursive_rollup.rb | 5 +- app/jobs/stat_collection_job.rb | 8 ++-- app/jobs/uri_collection_job.rb | 69 ++++++++++----------------- 3 files changed, 33 insertions(+), 49 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index 4d9e11b0..1da14c01 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -30,17 +30,16 @@ module RecursiveRollup # intervalo más amplio. # # @param :name [String] - # @param :beginning [Time] # @param :operation [Symbol] # @param :dimensions [Hash] # @return [nil] - def reduce_rollup(name, beginning, operation, **dimensions) + def reduce_rollup(name:, dimensions:, operation: :sum) Stat::INTERVALS.reduce do |previous, current| recursive_rollup(name: name, interval_previous: previous, interval: current, dimensions: dimensions, - beginning: beginning, + beginning: beginning_of_interval, operation: operation) # Devolver el intervalo actual diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 63d44999..e402e3b5 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -22,9 +22,11 @@ class StatCollectionJob < PeriodicJob rollup.average(:seconds) end - reduce_rollup('builds', beginning, :sum, site_id: site_id) - reduce_rollup('space_used', beginning, :average, site_id: site_id) - reduce_rollup('build_time', beginning, :average, site_id: site_id) + dimensions = { site_id: site_id } + + reduce_rollup(name: 'builds', operation: :sum, dimensions: dimensions) + reduce_rollup(name: 'space_used', operation: :average, dimensions: dimensions) + reduce_rollup(name: 'build_time', operation: :average, dimensions: dimensions) stat.touch run_again! unless once diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 90789aba..a8c51d84 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -22,7 +22,7 @@ class UriCollectionJob < PeriodicJob @site = Site.find site_id # Obtener el principio del intervalo anterior - beginning = beginning_of_interval + beginning_of_interval # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME # Columnas a agrupar @@ -33,51 +33,33 @@ class UriCollectionJob < PeriodicJob # rápido buscar en los rollups indexados que en la tabla en bruto. # # Los referers solo se agrupan por host. - site.hostnames.each do |host| - break if stop? + columns.each_key do |column| + columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) + end - host_dimensions = { host: host } - columns.each_key do |column| - columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) - end + # Las URIs son la fuente de verdad de las visitas, porque son las + # que indican las páginas y recursos descargables, el resto son + # imágenes, CSS, JS y tipografías que no nos aportan números + # significativos. + uri_dimensions = { host: site.hostnames, uri: uris } + host_dimensions = { host: site.hostnames } - # Las URIs son la fuente de verdad de las visitas, porque son las - # que indican las páginas y recursos descargables, el resto son - # imágenes, CSS, JS y tipografías que no nos aportan números - # significativos. - name = 'host|uri' - dimensions = { host: host, uri: uris } + # Cantidad de visitas por host + rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) + reduce_rollup(name: 'host', operation: :sum, dimensions: uri_dimensions) - rollup(name, beginning, **dimensions) - reduce_rollup(name, beginning, :sum, **dimensions) + # Cantidad de visitas por página/recurso + rollup(name: 'host|uri', dimensions: uri_dimensions) + reduce_rollup(name: 'host|uri', operation: :sum, dimensions: uri_dimensions) - columns.each_pair do |column, values| - # Obtener orígenes de visitas por host - values.each do |value| - column_name = "host|uri|#{column}" - column_dimensions = { host: host, uri: uris } - column_dimensions[column] = value + # Cantidad de visitas host y parámetro + columns.each_pair do |column, values| + column_name = "host|#{column}" + column_dimensions = { host: site.hostnames } + column_dimensions[column] = values - rollup(column_name, beginning, **column_dimensions) - reduce_rollup(column_name, beginning, :sum, **column_dimensions) - end - end - - # Reducir todas las visitas a cantidad de visitas por host - rollup('host', beginning, **host_dimensions) - - # Acumular por mes y año - reduce_rollup('host', beginning, :sum, **host_dimensions) - - columns.each_pair do |column, values| - values.each do |value| - column_dimensions = { host: host } - column_dimensions[column] = value - - rollup("host|#{column}", beginning, **column_dimensions) - reduce_rollup("host|#{column}", beginning, :sum, **column_dimensions) - end - end + rollup(name: column_name, dimensions: column_dimensions, filter: uri_dimensions) + reduce_rollup(name: column_name, dimensions: column_dimensions) end stat.touch @@ -92,9 +74,10 @@ class UriCollectionJob < PeriodicJob # @param :name [String] # @param :beginning [Time] # @param :dimensions [Hash] + # @param :filter [Hash] # @return [nil] - def rollup(name, beginning, **dimensions) - AccessLog.where(**dimensions) + def rollup(name:, beginning:, dimensions:, filter: nil) + AccessLog.where(**(filter || dimensions)) .where('created_at >= ?', beginning) .completed_requests .non_robots From 6464f35625f26ef1627682ad97ca4a143d0d8179 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 19:38:32 -0300 Subject: [PATCH 102/121] =?UTF-8?q?fixup!=20simplificar=20el=20c=C3=B3digo?= =?UTF-8?q?!!?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index a8c51d84..bab85d0d 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -28,6 +28,13 @@ class UriCollectionJob < PeriodicJob # Columnas a agrupar columns = Stat::COLUMNS.zip([nil]).to_h + # Las URIs son la fuente de verdad de las visitas, porque son las + # que indican las páginas y recursos descargables, el resto son + # imágenes, CSS, JS y tipografías que no nos aportan números + # significativos. + uri_dimensions = { host: site.hostnames, uri: uris } + host_dimensions = { host: site.hostnames } + # Recorremos todos los hostnames y uris posibles y luego agrupamos # recursivamente para no tener que recalcular, asumiendo que es más # rápido buscar en los rollups indexados que en la tabla en bruto. @@ -37,13 +44,6 @@ class UriCollectionJob < PeriodicJob columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) end - # Las URIs son la fuente de verdad de las visitas, porque son las - # que indican las páginas y recursos descargables, el resto son - # imágenes, CSS, JS y tipografías que no nos aportan números - # significativos. - uri_dimensions = { host: site.hostnames, uri: uris } - host_dimensions = { host: site.hostnames } - # Cantidad de visitas por host rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) reduce_rollup(name: 'host', operation: :sum, dimensions: uri_dimensions) From 8a3c9a76a72d2e384005f81af0013f3dad54e23e Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 19:44:02 -0300 Subject: [PATCH 103/121] =?UTF-8?q?fixup!=20fixup!=20simplificar=20el=20c?= =?UTF-8?q?=C3=B3digo!!?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index bab85d0d..ca755ca1 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -76,9 +76,9 @@ class UriCollectionJob < PeriodicJob # @param :dimensions [Hash] # @param :filter [Hash] # @return [nil] - def rollup(name:, beginning:, dimensions:, filter: nil) + def rollup(name:, dimensions:, filter: nil) AccessLog.where(**(filter || dimensions)) - .where('created_at >= ?', beginning) + .where('created_at >= ?', beginning_of_interval) .completed_requests .non_robots .group(*dimensions.keys) From 158c2244490fc79467d915e4841b6d54a34e4a49 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 20:40:54 -0300 Subject: [PATCH 104/121] generar los rollups para los siguientes intervalos --- app/jobs/uri_collection_job.rb | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index ca755ca1..0ea468da 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -11,8 +11,6 @@ # Los hostnames de un sitio van a poder obtenerse a partir de # Site#hostnames con la garantía de que son únicos. class UriCollectionJob < PeriodicJob - include RecursiveRollup - # Ignoramos imágenes porque suelen ser demasiadas y no aportan a las # estadísticas. IMAGES = %w[.png .jpg .jpeg .gif .webp .jfif].freeze @@ -76,13 +74,29 @@ class UriCollectionJob < PeriodicJob # @param :dimensions [Hash] # @param :filter [Hash] # @return [nil] - def rollup(name:, dimensions:, filter: nil) + def rollup(name:, dimensions:, interval: starting_interval, filter: nil) AccessLog.where(**(filter || dimensions)) .where('created_at >= ?', beginning_of_interval) .completed_requests .non_robots .group(*dimensions.keys) - .rollup(name, interval: starting_interval, update: true) + .rollup(name, interval: interval, update: true) + end + + # Generar rollups con el resto de la información + # + # @param :name [String] + # @param :dimensions [Hash] + # @param :filter [Hash] + # @return [nil] + def reduce_rollup(name:, dimensions:, filter: nil) + Stat::INTERVALS.reduce do |_previous, current| + rollup(name: name, dimensions: dimensions, filter: filter, interval: current) + + current + end + + nil end def stat_name From 3f29d87acdb91d59f79fd38f105ea038c1909a94 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 20:41:38 -0300 Subject: [PATCH 105/121] =?UTF-8?q?las=20tablas=20tambi=C3=A9n=20est=C3=A1?= =?UTF-8?q?n=20filtradas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 36b72721..c2eb0425 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -46,17 +46,17 @@ class StatsController < ApplicationController Stat::COLUMNS.each do |column| @columns[column] = Rails.cache.fetch("stats/#{column}/#{site.id}", expires_in: expires_in) do - Rollup.where(interval: interval, name: "host|#{column}") - .where_dimensions(host: hostnames) - .group("dimensions->>'#{column}'") - .sum(:value) - .transform_values(&:to_i) - .transform_values do |v| - v * nodes - end - .sort_by do |_, v| - v - end.reverse.to_h + rollup_scope.where(interval: interval, name: "host|#{column}") + .where_dimensions(host: hostnames) + .group("dimensions->>'#{column}'") + .sum(:value) + .transform_values(&:to_i) + .transform_values do |v| + v * nodes + end + .sort_by do |_, v| + v + end.reverse.to_h end end end From c8ff6de9ae5ab32ce58b754cbd449b9fb5182ce7 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 20:42:10 -0300 Subject: [PATCH 106/121] trasladar los parametros a los graficos --- app/controllers/stats_controller.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index c2eb0425..03d5fc9b 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -34,7 +34,14 @@ class StatsController < ApplicationController def index breadcrumb I18n.t('stats.index.title'), '' - @chart_params = { interval: interval } + params.with_defaults! default_url_options + + @chart_params = { + interval: interval, + period_start: params[:period_start], + period_end: params[:period_end] + } + hostnames last_stat chart_options From 62bdfff98c5ef174e8c7ed9246aee1fbd44f9f4d Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 20:42:38 -0300 Subject: [PATCH 107/121] poder ver y filtrar por rangos --- app/views/stats/index.haml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 6bbf596e..8987560f 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -9,9 +9,13 @@ %time{ datetime: @last_stat.updated_at } #{time_ago_in_words @last_stat.updated_at}. - .mb-5 + %form.mb-5.form-inline{ method: 'get' } - Stat::INTERVALS.each do |interval| - = link_to t(".#{interval}"), site_stats_path(interval: interval, urls: params[:urls]), class: "btn #{'btn-primary active' if @interval == interval}" + = link_to t(".#{interval}"), site_stats_path(interval: interval, urls: params[:urls], period_start: params[:period_start].to_date.try(:"beginning_of_#{interval}").to_date, period_end: params[:period_end]), class: "mb-0 btn #{'btn-primary active' if @interval == interval}" + + %input.form-control{ type: 'date', name: :period_start, value: params[:period_start] } + %input.form-control{ type: 'date', name: :period_end, value: params[:period_end] } + %button.btn.mb-0{ type: 'submit' }= t('.filter') .mb-5 %h2= t('.host.title', count: @hostnames.size) From 8ca5085aae92c72492a6ff6380f130aa0edc0012 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 20:48:53 -0300 Subject: [PATCH 108/121] fixup! generar los rollups para los siguientes intervalos --- app/jobs/uri_collection_job.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 0ea468da..42c05963 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -44,11 +44,11 @@ class UriCollectionJob < PeriodicJob # Cantidad de visitas por host rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) - reduce_rollup(name: 'host', operation: :sum, dimensions: uri_dimensions) + reduce_rollup(name: 'host', dimensions: uri_dimensions) # Cantidad de visitas por página/recurso rollup(name: 'host|uri', dimensions: uri_dimensions) - reduce_rollup(name: 'host|uri', operation: :sum, dimensions: uri_dimensions) + reduce_rollup(name: 'host|uri', dimensions: uri_dimensions) # Cantidad de visitas host y parámetro columns.each_pair do |column, values| From 35d16b11397c5ae5ee3a442cd30827080a7c4717 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 30 Apr 2022 21:21:58 -0300 Subject: [PATCH 109/121] fixup! fixup! generar los rollups para los siguientes intervalos --- app/jobs/uri_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 42c05963..59f753ec 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -44,7 +44,7 @@ class UriCollectionJob < PeriodicJob # Cantidad de visitas por host rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) - reduce_rollup(name: 'host', dimensions: uri_dimensions) + reduce_rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) # Cantidad de visitas por página/recurso rollup(name: 'host|uri', dimensions: uri_dimensions) From 755db601a161a5cba148ec6567959fa2c6b1dace Mon Sep 17 00:00:00 2001 From: f Date: Mon, 2 May 2022 13:41:37 -0300 Subject: [PATCH 110/121] ordenar en la base de datos --- app/controllers/stats_controller.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 03d5fc9b..cccdb8b6 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -56,14 +56,12 @@ class StatsController < ApplicationController rollup_scope.where(interval: interval, name: "host|#{column}") .where_dimensions(host: hostnames) .group("dimensions->>'#{column}'") + .order('sum(value) desc') .sum(:value) .transform_values(&:to_i) .transform_values do |v| v * nodes end - .sort_by do |_, v| - v - end.reverse.to_h end end end From 844ba75d08b213c42e128ea6578c91d91b8369e7 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 30 May 2022 15:10:52 -0300 Subject: [PATCH 111/121] =?UTF-8?q?indicar=20qu=C3=A9=20significa=20lo=20v?= =?UTF-8?q?ac=C3=ADo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/stats/index.haml | 2 +- config/locales/en.yml | 3 ++- config/locales/es.yml | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 8987560f..058f17ef 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -55,7 +55,7 @@ %tbody - values.each_pair do |col, val| %tr - %th{ scope: 'row', style: 'word-break: break-all' }= col.blank? ? t('.columns.empty') : col + %th{ scope: 'row', style: 'word-break: break-all' }= col.blank? ? t(".columns.#{column}.empty") : col %td= val .mb-5 %h2= t('.resources.title') diff --git a/config/locales/en.yml b/config/locales/en.yml index ec8a5792..87a18ff6 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -288,15 +288,16 @@ en: description: 'Average time your site takes to build.' columns: visits: "Visits" - empty: "(nothing)" http_referer: title: "Referers" description: "Visits by origin" column: "Referer" + empty: "(direct visit)" geoip2_data_country_name: title: "Countries" description: "Visits by country" column: "Country" + empty: "(couldn't detect country)" sites: donations: url: 'https://donaciones.sutty.nl/en/' diff --git a/config/locales/es.yml b/config/locales/es.yml index 30ea15fc..3c9d11b5 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -293,15 +293,16 @@ es: description: 'Tiempo promedio que toma en publicarse tu sitio.' columns: visits: "Visitas" - empty: "(nada)" http_referer: title: "Referencias" description: "Orígenes de las visitas" column: "Referencia" + empty: "(visita directa)" geoip2_data_country_name: title: "Países" description: "Cantidad de visitas por país" column: "País" + empty: "(no se pudo detectar el país)" sites: donations: url: 'https://donaciones.sutty.nl/' From 472085301960d6829aa4c1d195cda4bc96fcac78 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 11 Jun 2022 19:33:02 -0300 Subject: [PATCH 112/121] =?UTF-8?q?al=20recargar=20la=20p=C3=A1gina=20volv?= =?UTF-8?q?er=20al=20gr=C3=A1fico?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/stats/index.haml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 058f17ef..49fbd023 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -22,10 +22,10 @@ %p.lead= t('.host.description') = line_chart site_stats_host_path(@chart_params), **@chart_options - .mb-5 + #custom-urls.mb-5 %h2= t('.urls.title') %p.lead= t('.urls.description') - %form + %form{ method: 'get', action: '#custom-urls' } %input{ type: 'hidden', name: 'interval', value: @interval } .form-group %label{ for: 'urls' }= t('.urls.label') From 6f45756e6d6b5ea12386288821fd45d6b4ebed1b Mon Sep 17 00:00:00 2001 From: f Date: Sat, 11 Jun 2022 19:33:21 -0300 Subject: [PATCH 113/121] =?UTF-8?q?mejorar=20el=20mensaje=20de=20tiempo=20?= =?UTF-8?q?de=20compilaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/locales/en.yml | 2 +- config/locales/es.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index 87a18ff6..f89778ed 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -285,7 +285,7 @@ en: description: 'Average storage space used by your site.' build_time: title: 'Publication time' - description: 'Average time your site takes to build.' + description: 'Average time your site takes to build, from pressing "Publish changes" to actually being available on your site.' columns: visits: "Visits" http_referer: diff --git a/config/locales/es.yml b/config/locales/es.yml index 3c9d11b5..bf3e93f0 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -290,7 +290,7 @@ es: description: 'Espacio en disco que ocupa en promedio tu sitio.' build_time: title: 'Tiempo de publicación' - description: 'Tiempo promedio que toma en publicarse tu sitio.' + description: 'Tiempo que tarda el sitio en generarse, desde que usas el botón "Publicar cambios" hasta que los puedes ver en el sitio' columns: visits: "Visitas" http_referer: From 00f303cf20f8b7338e402eac0d9baf8465b822e5 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 11 Jun 2022 19:42:55 -0300 Subject: [PATCH 114/121] =?UTF-8?q?procesar=20las=20estad=C3=ADsticas=20au?= =?UTF-8?q?tom=C3=A1ticamente?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Procfile | 1 + lib/tasks/stats.rake | 11 +++++++++++ monit.conf | 5 +++++ 3 files changed, 17 insertions(+) create mode 100644 lib/tasks/stats.rake diff --git a/Procfile b/Procfile index b308ffd5..8f6c7741 100644 --- a/Procfile +++ b/Procfile @@ -5,3 +5,4 @@ blazer_1h: bundle exec rake blazer:run_checks SCHEDULE="1 hour" blazer_1d: bundle exec rake blazer:run_checks SCHEDULE="1 day" blazer: bundle exec rake blazer:send_failing_checks prometheus: bundle exec prometheus_exporter -b 0.0.0.0 --prefix "sutty_" +stats: bundle exec rake stats:process_all diff --git a/lib/tasks/stats.rake b/lib/tasks/stats.rake new file mode 100644 index 00000000..9461782a --- /dev/null +++ b/lib/tasks/stats.rake @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +namespace :stats do + desc 'Process stats' + task process_all: :environment do + Site.all.pluck(:id).each do |site_id| + UriCollectionJob.perform_now site_id: site_id, once: true + StatCollectionJob.perform_now site_id: site_id, once: true + end + end +end diff --git a/monit.conf b/monit.conf index 28db0cf0..83d17449 100644 --- a/monit.conf +++ b/monit.conf @@ -30,3 +30,8 @@ check program access_logs with path "/srv/http/bin/access_logs" as uid "app" and gid "www-data" every "0 0 * * *" if status != 0 then alert + +check program stats + with path "/usr/bin/foreman run -f /srv/Procfile -d /srv stats" as uid "rails" gid "www-data" + every "0 1 * * *" + if status != 0 then alert From dd34d176ea53cd24b157ce3aaf5da238f9c47217 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 15 Jun 2022 13:34:46 -0300 Subject: [PATCH 115/121] sincronizar links duros --- app/models/deploy_rsync.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb index b2ffba7c..996f8cdd 100644 --- a/app/models/deploy_rsync.rb +++ b/app/models/deploy_rsync.rb @@ -84,7 +84,7 @@ class DeployRsync < Deploy # # @return [Boolean] def rsync - run %(rsync -avi --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + run %(rsync -aviH --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) end # El origen es el destino de la compilación From 655c6b65d59c37be310c4c48f0b4c2cad7939a9c Mon Sep 17 00:00:00 2001 From: f Date: Fri, 1 Jul 2022 14:43:30 -0300 Subject: [PATCH 116/121] procesar todas las urls a veces las urls vienen como arrays, con lo que no hay que procesarlas. --- app/controllers/stats_controller.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index cccdb8b6..c2c7bc58 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -133,7 +133,8 @@ class StatsController < ApplicationController def normalized_urls @normalized_urls ||= begin - urls = params.permit(:urls).try(:[], :urls)&.split("\n")&.map(&:strip)&.select(&:present?)&.select do |uri| + urls = params[:urls].is_a?(Array) ? params[:urls] : params[:urls]&.split("\n") + urls = urls&.map(&:strip)&.select(&:present?)&.select do |uri| uri.start_with? 'https://' end From e6c03df13f9bf7d0d6e27e1fbc7cecb54cd280f0 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 1 Jul 2022 15:01:57 -0300 Subject: [PATCH 117/121] =?UTF-8?q?mostrar=20las=20estadisticas=20en=20el?= =?UTF-8?q?=20men=C3=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/posts/index.haml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index accfdb34..c6ffc2f5 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -21,6 +21,9 @@ - if policy(@site).private? = link_to t('sites.private'), '../private/' + @site.name, class: 'btn', target: '_blank', rel: 'noopener' + - if policy(SiteStat.new(@site)).index? + = link_to t('stats.index.title'), site_stats_path(@site), class: 'btn' + - if policy(SiteUsuarie.new(@site, current_usuarie)).index? = render 'layouts/btn_with_tooltip', tooltip: t('usuaries.index.help.self'), From f1137c6ed28c5245f27865c489904a1012f119b3 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 1 Jul 2022 15:02:17 -0300 Subject: [PATCH 118/121] error de sintaxis --- config/application.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/config/application.rb b/config/application.rb index 7c530e69..f1bf8365 100644 --- a/config/application.rb +++ b/config/application.rb @@ -38,6 +38,7 @@ module Sutty config.active_storage.variant_processor = :vips + config.to_prepare do # Load application's model / class decorators Dir.glob(File.join(File.dirname(__FILE__), '..', 'app', '**', '*_decorator.rb')).sort.each do |c| Rails.configuration.cache_classes ? require(c) : load(c) From 332ae66b8a54636b9bafea7f992719deb6af8122 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 15 Jul 2022 13:19:08 -0300 Subject: [PATCH 119/121] =?UTF-8?q?no=20juntar=20estad=C3=ADsticas=20por?= =?UTF-8?q?=20mes=20y=20a=C3=B1o=20todav=C3=ADa?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit la forma en que las estamos recopilando es rápida pero no acumulativa, entonces los números siempre dan mal. --- app/models/stat.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/stat.rb b/app/models/stat.rb index f1016026..6c681aa4 100644 --- a/app/models/stat.rb +++ b/app/models/stat.rb @@ -3,7 +3,7 @@ # Registran cuándo fue la última recolección de datos. class Stat < ApplicationRecord # XXX: Los intervalos van en orden de mayor especificidad a menor - INTERVALS = %i[day month year].freeze + INTERVALS = %i[day].freeze RESOURCES = %i[builds space_used build_time].freeze COLUMNS = %i[http_referer geoip2_data_country_name].freeze From 091ceac788595edf537d81ffd929c0dc5084f230 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 15 Jul 2022 14:17:46 -0300 Subject: [PATCH 120/121] =?UTF-8?q?bot=C3=B3n=20de=20estad=C3=ADsticas=20m?= =?UTF-8?q?=C3=A1s=20arriba?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/posts_controller.rb | 2 ++ app/views/posts/index.haml | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index e6f836bb..c5dc0f54 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -35,6 +35,8 @@ class PostsController < ApplicationController # Filtrar los posts que les invitades no pueden ver @usuarie = site.usuarie? current_usuarie + + @site_stat = SiteStat.new(site) end def show diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index c6ffc2f5..bc5c826c 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -15,15 +15,15 @@ - else %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'btn btn-secondary btn-sm' + - if policy(@site_stat).index? + = link_to t('stats.index.title'), site_stats_path(@site), class: 'btn' + - if policy(@site).edit? = link_to t('sites.edit.btn', site: @site.title), edit_site_path(@site), class: 'btn' - if policy(@site).private? = link_to t('sites.private'), '../private/' + @site.name, class: 'btn', target: '_blank', rel: 'noopener' - - if policy(SiteStat.new(@site)).index? - = link_to t('stats.index.title'), site_stats_path(@site), class: 'btn' - - if policy(SiteUsuarie.new(@site, current_usuarie)).index? = render 'layouts/btn_with_tooltip', tooltip: t('usuaries.index.help.self'), From cacb355b620cb5547ca21ec3743aa15c5a6d8834 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 3 Aug 2022 13:24:36 -0300 Subject: [PATCH 121/121] los deploys pueden no tener destino closes #7151 closes #7152 closes #7153 closes #7154 closes #7167 closes #7168 closes #7169 closes #7170 closes #7380 closes #7381 closes #7382 closes #7383 closes #7394 closes #7395 closes #7396 closes #7397 closes #7405 closes #7406 closes #7407 closes #7408 closes #7491 closes #7492 closes #7493 closes #7494 closes #7509 closes #7510 closes #7511 closes #7512 closes #7527 closes #7528 closes #7529 closes #7530 --- app/jobs/uri_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 59f753ec..4cbbf593 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -109,7 +109,7 @@ class UriCollectionJob < PeriodicJob # # TODO: Cambiar al mergear origin-referer def destinations - @destinations ||= site.deploys.map(&:destination).select do |d| + @destinations ||= site.deploys.map(&:destination).compact.select do |d| File.directory?(d) end.map do |d| File.realpath(d)