From 094a8092de7721edab16cfb052db058dbe3464cd Mon Sep 17 00:00:00 2001 From: f Date: Fri, 3 Sep 2021 15:23:11 -0300 Subject: [PATCH 01/52] Agregar rollups --- Gemfile | 1 + Gemfile.lock | 10 +++++++++- db/migrate/20210807003928_create_rollups.rb | 15 +++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20210807003928_create_rollups.rb diff --git a/Gemfile b/Gemfile index 4256e307..2f5446a0 100644 --- a/Gemfile +++ b/Gemfile @@ -58,6 +58,7 @@ gem 'rails-i18n' gem 'rails_warden' gem 'redis', require: %w[redis redis/connection/hiredis] gem 'redis-rails' +gem 'rollups' gem 'rubyzip' gem 'rugged' gem 'concurrent-ruby-ext' diff --git a/Gemfile.lock b/Gemfile.lock index 33fba3a0..6c873d9b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -205,6 +205,8 @@ GEM ffi (~> 1.0) globalid (0.4.2) activesupport (>= 4.2.0) + groupdate (5.2.2) + activesupport (>= 5) hairtrigger (0.2.24) activerecord (>= 5.0, < 7) ruby2ruby (~> 2.4) @@ -345,6 +347,7 @@ GEM mini_histogram (0.3.1) mini_magick (4.11.0) mini_mime (1.1.0) + mini_portile2 (2.5.3) minima (2.5.1) jekyll (>= 3.5, < 5.0) jekyll-feed (~> 0.9) @@ -357,7 +360,8 @@ GEM net-ssh (6.1.0) netaddr (2.0.4) nio4r (2.5.7-x86_64-linux-musl) - nokogiri (1.11.7-x86_64-linux) + nokogiri (1.11.7-x86_64-linux-musl) + mini_portile2 (~> 2.5.0) racc (~> 1.4) orm_adapter (0.5.0) parallel (1.20.1) @@ -479,6 +483,9 @@ GEM actionpack (>= 5.0) railties (>= 5.0) rexml (3.2.5) + rollups (0.1.2) + activesupport (>= 5.1) + groupdate (>= 5.2) rouge (3.26.0) rubocop (1.18.3) parallel (~> 1.10) @@ -692,6 +699,7 @@ DEPENDENCIES recursero-jekyll-theme redis redis-rails + rollups rubocop-rails rubyzip rugged diff --git a/db/migrate/20210807003928_create_rollups.rb b/db/migrate/20210807003928_create_rollups.rb new file mode 100644 index 00000000..932513a4 --- /dev/null +++ b/db/migrate/20210807003928_create_rollups.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +# Crear la tabla de Rollups +class CreateRollups < ActiveRecord::Migration[6.1] + def change + create_table :rollups do |t| + t.string :name, null: false + t.string :interval, null: false + t.datetime :time, null: false + t.jsonb :dimensions, null: false, default: {} + t.float :value + end + add_index :rollups, %i[name interval time dimensions], unique: true + end +end From 604c16bfb8947b42634bc3f2f8c30f168fecee13 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 3 Sep 2021 15:28:45 -0300 Subject: [PATCH 02/52] Agregar created_at a access_logs para poder agrupar por fecha --- ...10807004941_add_create_at_to_access_logs.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 db/migrate/20210807004941_add_create_at_to_access_logs.rb diff --git a/db/migrate/20210807004941_add_create_at_to_access_logs.rb b/db/migrate/20210807004941_add_create_at_to_access_logs.rb new file mode 100644 index 00000000..0e106061 --- /dev/null +++ b/db/migrate/20210807004941_add_create_at_to_access_logs.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# Cambia los msec a datetime para poder agregar por tiempos +class AddCreateAtToAccessLogs < ActiveRecord::Migration[6.1] + def up + add_column :access_logs, :created_at, :datetime, precision: 6 + + create_trigger(compatibility: 1).on(:access_logs).before(:insert) do + 'new.created_at := to_timestamp(new.msec)' + end + + ActiveRecord::Base.connection.execute('update access_logs set created_at = to_timestamp(msec);') + end + + def down + remove_column :access_logs, :created_at + end +end From 016da2852926436142e89fcdf1f80c840496a084 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 3 Sep 2021 15:53:45 -0300 Subject: [PATCH 03/52] Inflexiones para Rollup --- config/initializers/inflections.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/initializers/inflections.rb b/config/initializers/inflections.rb index 35b309ea..0e18b987 100644 --- a/config/initializers/inflections.rb +++ b/config/initializers/inflections.rb @@ -11,6 +11,8 @@ ActiveSupport::Inflector.inflections(:en) do |inflect| inflect.singular 'licencias', 'licencia' inflect.plural 'rol', 'roles' inflect.singular 'roles', 'rol' + inflect.plural 'rollup', 'rollups' + inflect.singular 'rollups', 'rollup' end ActiveSupport::Inflector.inflections(:es) do |inflect| @@ -24,4 +26,6 @@ ActiveSupport::Inflector.inflections(:es) do |inflect| inflect.singular 'roles', 'rol' inflect.plural 'licencia', 'licencias' inflect.singular 'licencias', 'licencia' + inflect.plural 'rollup', 'rollups' + inflect.singular 'rollups', 'rollup' end From 4e4b5888a35e403133c8d9a0934a09cd6520f9ae Mon Sep 17 00:00:00 2001 From: f Date: Fri, 3 Sep 2021 15:55:30 -0300 Subject: [PATCH 04/52] Peticiones completas --- app/models/access_log.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/models/access_log.rb b/app/models/access_log.rb index 85cd4c36..0dceb7d7 100644 --- a/app/models/access_log.rb +++ b/app/models/access_log.rb @@ -1,4 +1,9 @@ # frozen_string_literal: true class AccessLog < ApplicationRecord + # Las peticiones completas son las que terminaron bien y se + # respondieron con 200 OK o 304 Not Modified + # + # @see {https://en.wikipedia.org/wiki/List_of_HTTP_status_codes} + scope :completed_requests, -> { where(request_method: 'GET', request_completion: 'OK', status: [200, 304]) } end From 662b5eeec4da85d4ae9fdd89f21e13501d6ff940 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 4 Oct 2021 14:48:19 -0300 Subject: [PATCH 05/52] guardar y mostrar la url actual en el campo permalink --- app/models/metadata_permalink.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/models/metadata_permalink.rb b/app/models/metadata_permalink.rb index 58feb9e5..dc65aa86 100644 --- a/app/models/metadata_permalink.rb +++ b/app/models/metadata_permalink.rb @@ -2,6 +2,12 @@ # Este metadato permite generar rutas manuales. class MetadataPermalink < MetadataString + # El valor por defecto una vez creado es la URL que le asigne Jekyll, + # de forma que nunca cambia aunque se cambie el título. + def default_value + document.url unless post.new? + end + # Los permalinks nunca pueden ser privados def private? false From 930f88903e5c73c0f5936f3124c34c9605bca837 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 7 Oct 2021 15:24:21 -0300 Subject: [PATCH 06/52] =?UTF-8?q?instalar=20chartkick=20para=20generar=20g?= =?UTF-8?q?r=C3=A1ficos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Gemfile | 1 + Gemfile.lock | 1 + package.json | 4 +++- yarn.lock | 24 ++++++++++++++++++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 2f5446a0..b5e8664b 100644 --- a/Gemfile +++ b/Gemfile @@ -28,6 +28,7 @@ gem 'jbuilder', '~> 2.5' # Use ActiveModel has_secure_password gem 'bcrypt', '~> 3.1.7' gem 'blazer' +gem 'chartkick' gem 'commonmarker' gem 'devise' gem 'devise-i18n' diff --git a/Gemfile.lock b/Gemfile.lock index 6c873d9b..9d1f512b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -642,6 +642,7 @@ DEPENDENCIES bootstrap (~> 4) brakeman capybara (~> 2.13) + chartkick commonmarker concurrent-ruby-ext database_cleaner diff --git a/package.json b/package.json index 0a2458a6..184f7ebb 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,8 @@ "zepto": "^1.2.0" }, "devDependencies": { - "@types/rails__activestorage": "^6.0.0" + "@types/rails__activestorage": "^6.0.0", + "chart.js": "^3.5.1", + "chartkick": "^4.0.5" } } diff --git a/yarn.lock b/yarn.lock index 11ff78cb..68b7fd23 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2119,6 +2119,25 @@ chalk@^4.1.0: ansi-styles "^4.1.0" supports-color "^7.1.0" +chart.js@>=3.0.2, chart.js@^3.5.1: + version "3.5.1" + resolved "https://registry.yarnpkg.com/chart.js/-/chart.js-3.5.1.tgz#73e24d23a4134a70ccdb5e79a917f156b6f3644a" + integrity sha512-m5kzt72I1WQ9LILwQC4syla/LD/N413RYv2Dx2nnTkRS9iv/ey1xLTt0DnPc/eWV4zI+BgEgDYBIzbQhZHc/PQ== + +chartjs-adapter-date-fns@>=2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/chartjs-adapter-date-fns/-/chartjs-adapter-date-fns-2.0.0.tgz#5e53b2f660b993698f936f509c86dddf9ed44c6b" + integrity sha512-rmZINGLe+9IiiEB0kb57vH3UugAtYw33anRiw5kS2Tu87agpetDDoouquycWc9pRsKtQo5j+vLsYHyr8etAvFw== + +chartkick@^4.0.5: + version "4.0.5" + resolved "https://registry.yarnpkg.com/chartkick/-/chartkick-4.0.5.tgz#310a60c931e8ceedc39adee2ef8e9d1e474cb0e6" + integrity sha512-xKak4Fsgfvp1hj/LykRKkniDMaZASx2A4TdVc/sfsiNFFNf1m+D7PGwP1vgj1UsbsCjOCSfGWWyJpOYxkUCBug== + optionalDependencies: + chart.js ">=3.0.2" + chartjs-adapter-date-fns ">=2.0.0" + date-fns ">=2.0.0" + chokidar@^2.1.8: version "2.1.8" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.8.tgz#804b3a7b6a99358c3c5c61e71d8728f041cff917" @@ -2744,6 +2763,11 @@ dashdash@^1.12.0: dependencies: assert-plus "^1.0.0" +date-fns@>=2.0.0: + version "2.24.0" + resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.24.0.tgz#7d86dc0d93c87b76b63d213b4413337cfd1c105d" + integrity sha512-6ujwvwgPID6zbI0o7UbURi2vlLDR9uP26+tW6Lg+Ji3w7dd0i3DOcjcClLjLPranT60SSEFBwdSyYwn/ZkPIuw== + debug@2.6.9, debug@^2.2.0, debug@^2.3.3: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" From a7ae7f8e8d7cec044d2d01a49b51bc100e629e23 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 8 Oct 2021 16:31:02 -0300 Subject: [PATCH 07/52] ver cantidad de visitas --- app/controllers/stats_controller.rb | 64 ++++++++++++++++++++++++++--- app/javascript/packs/application.js | 1 + app/policies/site_stat_policy.rb | 4 ++ app/views/stats/index.haml | 19 ++++----- config/locales/en.yml | 9 ++++ config/locales/es.yml | 13 ++++++ config/routes.rb | 1 + 7 files changed, 94 insertions(+), 17 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 07baaf1a..07be68c4 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -5,14 +5,68 @@ class StatsController < ApplicationController include Pundit before_action :authenticate_usuarie! + INTERVALS = %i[hour day week month].freeze + + # XXX: Permitir a Chart.js inyectar su propio CSS + content_security_policy only: :index do |policy| + policy.style_src :self, :unsafe_inline + policy.script_src :self, :unsafe_inline + end + def index @site = find_site authorize SiteStat.new(@site) - # Solo queremos el promedio de tiempo de compilación, no de - # instalación de dependencias. - stats = @site.build_stats.jekyll - @build_avg = stats.average(:seconds).to_f.round(2) - @build_max = stats.maximum(:seconds).to_f.round(2) + @chart_params = { interval: interval } + hostnames + end + + # Genera un gráfico de visitas por dominio asociado a este sitio + def host + @site = find_site + authorize SiteStat.new(@site) + + @stats = Rollup.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series| + series.each do |serie| + serie[:name] = serie.dig(:dimensions, 'host') + serie[:data].transform_values! do |value| + value * nodes + end + end + end + + render json: @stats + end + + private + + # TODO: Eliminar cuando mergeemos referer-origin + def hostnames + @hostnames ||= [@site.hostname, @site.alternative_hostnames].flatten + end + + # Obtiene y valida los intervalos + # + # @return [Symbol] + def interval + @interval ||= begin + i = params[:interval].to_sym + INTERVALS.include?(i) ? i : :day + end + end + + # Obtiene la cantidad de nodos de Sutty, para poder calcular la + # cantidad de visitas. + # + # Como repartimos las visitas por nodo rotando las IPs en el + # nameserver y los resolvedores de DNS eligen un nameserver + # aleatoriamente, la cantidad de visitas se reparte + # equitativamente. + # + # XXX: Remover cuando podamos centralizar los AccessLog + # + # @return [Integer] + def nodes + @nodes ||= ENV.fetch('NODES', 1).to_i end end diff --git a/app/javascript/packs/application.js b/app/javascript/packs/application.js index 6aa3a2e1..492ca736 100644 --- a/app/javascript/packs/application.js +++ b/app/javascript/packs/application.js @@ -18,6 +18,7 @@ import 'etc' import Rails from '@rails/ujs' import Turbolinks from 'turbolinks' import * as ActiveStorage from '@rails/activestorage' +import 'chartkick/chart.js' Rails.start() Turbolinks.start() diff --git a/app/policies/site_stat_policy.rb b/app/policies/site_stat_policy.rb index a797034c..d11be95d 100644 --- a/app/policies/site_stat_policy.rb +++ b/app/policies/site_stat_policy.rb @@ -12,4 +12,8 @@ class SiteStatPolicy def index? site_stat.site.usuarie? usuarie end + + def host? + index? + end end diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index f49cdd15..b5943690 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -1,17 +1,12 @@ -= render 'layouts/breadcrumb', - crumbs: [link_to(t('sites.index.title'), sites_path), - link_to(@site.name, site_path(@site)), t('.title')] - .row .col %h1= t('.title') %p.lead= t('.help') - %table.table.table-condensed - %tbody - %tr - %td= t('.build.average') - %td= distance_of_time_in_words_if_more_than_a_minute @build_avg - %tr - %td= t('.build.maximum') - %td= distance_of_time_in_words_if_more_than_a_minute @build_max + .mb-3 + - StatsController::INTERVALS.each do |interval| + = link_to t(".#{interval}"), site_stats_path(interval: interval), class: 'btn' + + .mb-3 + %h2= t('.host', count: @hostnames.size) + = line_chart site_stats_host_path(@chart_params), locale: 'es' diff --git a/config/locales/en.yml b/config/locales/en.yml index fc194eab..96250301 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -255,6 +255,15 @@ en: build: average: 'Average building time' maximum: 'Maximum building time' + hour: 'Hourly' + day: 'Daily' + week: 'Weekly' + month: 'Monthly' + year: 'Yearly' + host: + zero: 'Site visits' + one: 'Site visits' + other: 'Visits by site name' sites: donations: url: 'https://donaciones.sutty.nl/en/' diff --git a/config/locales/es.yml b/config/locales/es.yml index e8185391..7fcebcbe 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -260,6 +260,15 @@ es: build: average: 'Tiempo promedio de generación' maximum: 'Tiempo máximo de generación' + hour: 'Por hora' + day: 'Diarias' + week: 'Semanales' + month: 'Mensuales' + year: 'Anuales' + host: + zero: 'Visitas del sitio' + one: 'Visitas del sitio' + other: 'Visitas por nombre del sitio' sites: donations: url: 'https://donaciones.sutty.nl/' @@ -584,3 +593,7 @@ es: edit: 'Editando' usuaries: index: 'Usuaries' + day: 'Día' + week: 'Semana' + month: 'Mes' + year: 'Año' diff --git a/config/routes.rb b/config/routes.rb index 2c5f1c60..0af58080 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -75,5 +75,6 @@ Rails.application.routes.draw do post 'reorder_posts', to: 'sites#reorder_posts' resources :stats, only: [:index] + get :'stats/host', to: 'stats#host' end end From df2b66afe85d806e32aee341fdc4beb5ee5cd873 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 8 Oct 2021 18:21:09 -0300 Subject: [PATCH 08/52] =?UTF-8?q?llevar=20el=20registro=20de=20las=20recol?= =?UTF-8?q?ecciones=20de=20estad=C3=ADsticas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/stat.rb | 3 +++ db/migrate/20211008201239_create_stats.rb | 12 ++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 app/models/stat.rb create mode 100644 db/migrate/20211008201239_create_stats.rb diff --git a/app/models/stat.rb b/app/models/stat.rb new file mode 100644 index 00000000..cf717b05 --- /dev/null +++ b/app/models/stat.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +class Stat < ApplicationRecord; end diff --git a/db/migrate/20211008201239_create_stats.rb b/db/migrate/20211008201239_create_stats.rb new file mode 100644 index 00000000..e1aff8f6 --- /dev/null +++ b/db/migrate/20211008201239_create_stats.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +# Una tabla que lleva el recuento de recolección de estadísticas, solo +# es necesario para saber cuándo se hicieron, si se hicieron y usar como +# caché. +class CreateStats < ActiveRecord::Migration[6.1] + def change + create_table :stats do |t| + t.timestamps + end + end +end From a2e4e0ab89910f85632ad231bbc2bdc01e69836b Mon Sep 17 00:00:00 2001 From: f Date: Fri, 8 Oct 2021 18:24:19 -0300 Subject: [PATCH 09/52] =?UTF-8?q?informar=20hace=20cu=C3=A1nto=20se=20actu?= =?UTF-8?q?alizaron=20los=20datos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 5 +++++ app/views/stats/index.haml | 5 +++++ config/locales/en.yml | 4 +--- config/locales/es.yml | 4 +--- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 07be68c4..8d1575bd 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -19,6 +19,7 @@ class StatsController < ApplicationController @chart_params = { interval: interval } hostnames + last_stat end # Genera un gráfico de visitas por dominio asociado a este sitio @@ -40,6 +41,10 @@ class StatsController < ApplicationController private + def last_stat + @last_stat ||= Stat.last + end + # TODO: Eliminar cuando mergeemos referer-origin def hostnames @hostnames ||= [@site.hostname, @site.alternative_hostnames].flatten diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index b5943690..e77454b1 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -2,6 +2,11 @@ .col %h1= t('.title') %p.lead= t('.help') + %p + %small + = t('.last_update') + %time{ datetime: @last_stat.created_at } + "#{time_ago_in_words @last_stat.created_at}." .mb-3 - StatsController::INTERVALS.each do |interval| diff --git a/config/locales/en.yml b/config/locales/en.yml index 96250301..623d8c2e 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -252,9 +252,7 @@ en: help: | These statistics show information about how your site is generated and how many resources it uses. - build: - average: 'Average building time' - maximum: 'Maximum building time' + last_update: 'Updated every hour. Last update on ' hour: 'Hourly' day: 'Daily' week: 'Weekly' diff --git a/config/locales/es.yml b/config/locales/es.yml index 7fcebcbe..4085a862 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -257,9 +257,7 @@ es: help: | Las estadísticas visibilizan información sobre cómo se genera y cuántos recursos utiliza tu sitio. - build: - average: 'Tiempo promedio de generación' - maximum: 'Tiempo máximo de generación' + last_update: 'Actualizadas cada hora. Última actualización hace ' hour: 'Por hora' day: 'Diarias' week: 'Semanales' From cfb2d7a61d1c29da08d9dd000d7d9039fe2af54e Mon Sep 17 00:00:00 2001 From: f Date: Fri, 8 Oct 2021 18:33:31 -0300 Subject: [PATCH 10/52] todas las peticiones necesitan une usuarie --- app/controllers/stats_controller.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 8d1575bd..0c185329 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -4,6 +4,7 @@ class StatsController < ApplicationController include Pundit before_action :authenticate_usuarie! + before_action :authorize_stats INTERVALS = %i[hour day week month].freeze @@ -14,9 +15,6 @@ class StatsController < ApplicationController end def index - @site = find_site - authorize SiteStat.new(@site) - @chart_params = { interval: interval } hostnames last_stat @@ -24,8 +22,6 @@ class StatsController < ApplicationController # Genera un gráfico de visitas por dominio asociado a este sitio def host - @site = find_site - authorize SiteStat.new(@site) @stats = Rollup.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series| series.each do |serie| @@ -45,6 +41,11 @@ class StatsController < ApplicationController @last_stat ||= Stat.last end + def authorize_stats + @site = find_site + authorize SiteStat.new(@site) + end + # TODO: Eliminar cuando mergeemos referer-origin def hostnames @hostnames ||= [@site.hostname, @site.alternative_hostnames].flatten From 38090d7de72d5cfdb1cad2e4349479ed17f04221 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 8 Oct 2021 18:34:16 -0300 Subject: [PATCH 11/52] resaltar el intervalo seleccionado --- app/views/stats/index.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index e77454b1..63aa76cd 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -10,7 +10,7 @@ .mb-3 - StatsController::INTERVALS.each do |interval| - = link_to t(".#{interval}"), site_stats_path(interval: interval), class: 'btn' + = link_to t(".#{interval}"), site_stats_path(interval: interval), class: "btn #{'btn-primary active' if params[:interval].to_sym == interval}" .mb-3 %h2= t('.host', count: @hostnames.size) From d2b1220df6f74ef46486beeb49ce9b1fd833658d Mon Sep 17 00:00:00 2001 From: f Date: Fri, 8 Oct 2021 18:35:40 -0300 Subject: [PATCH 12/52] informar cuando no hay datos --- app/controllers/stats_controller.rb | 24 ++++++++++++++++++++++++ config/locales/en.yml | 2 ++ config/locales/es.yml | 2 ++ 3 files changed, 28 insertions(+) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 0c185329..9cd1e5de 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -3,6 +3,8 @@ # Estadísticas del sitio class StatsController < ApplicationController include Pundit + include ActionView::Helpers::DateHelper + before_action :authenticate_usuarie! before_action :authorize_stats @@ -18,6 +20,7 @@ class StatsController < ApplicationController @chart_params = { interval: interval } hostnames last_stat + chart_options end # Genera un gráfico de visitas por dominio asociado a este sitio @@ -51,6 +54,27 @@ class StatsController < ApplicationController @hostnames ||= [@site.hostname, @site.alternative_hostnames].flatten end + # Opciones por defecto para los gráficos. + # + # La invitación a volver dentro de X tiempo es para dar un estimado de + # cuándo habrá información disponible, porque Rollup genera intervalos + # completos (¿aunque dice que no?) + # + # La diferencia se calcula sumando el intervalo a la hora de última + # toma de estadísticas y restando el tiempo que pasó desde ese + # momento. + def chart_options + time = last_stat.created_at + (1.send(interval)) + please_return_at = { please_return_at: distance_of_time_in_words(Time.now, time) } + + @chart_options ||= { + locale: I18n.locale, + empty: I18n.t('stats.index.empty', **please_return_at), + loading: I18n.t('stats.index.loading'), + html: %(
%{loading}
) + } + end + # Obtiene y valida los intervalos # # @return [Symbol] diff --git a/config/locales/en.yml b/config/locales/en.yml index 623d8c2e..58e0e726 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -253,6 +253,8 @@ en: These statistics show information about how your site is generated and how many resources it uses. last_update: 'Updated every hour. Last update on ' + empty: 'There is no enough information yet. We invite you to come back in %{please_return_at}!' + loading: 'Loading...' hour: 'Hourly' day: 'Daily' week: 'Weekly' diff --git a/config/locales/es.yml b/config/locales/es.yml index 4085a862..8df68896 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -258,6 +258,8 @@ es: Las estadísticas visibilizan información sobre cómo se genera y cuántos recursos utiliza tu sitio. last_update: 'Actualizadas cada hora. Última actualización hace ' + empty: 'Todavía no hay información suficiente. Te invitamos a volver en %{please_return_at} :)' + loading: 'Cargando...' hour: 'Por hora' day: 'Diarias' week: 'Semanales' From ef0055db0535492b62a95f6cd85970ead819ddb3 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 8 Oct 2021 18:36:31 -0300 Subject: [PATCH 13/52] =?UTF-8?q?agrupar=20por=20a=C3=B1o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 9cd1e5de..a0937477 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -8,7 +8,7 @@ class StatsController < ApplicationController before_action :authenticate_usuarie! before_action :authorize_stats - INTERVALS = %i[hour day week month].freeze + INTERVALS = %i[hour day week month year].freeze # XXX: Permitir a Chart.js inyectar su propio CSS content_security_policy only: :index do |policy| From 8e9401036ce4f3c64aff0db21263d13e56e14e3b Mon Sep 17 00:00:00 2001 From: f Date: Fri, 8 Oct 2021 18:38:02 -0300 Subject: [PATCH 14/52] =?UTF-8?q?cachear=20los=20resultados=20hasta=20la?= =?UTF-8?q?=20pr=C3=B3xima=20actualizaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index a0937477..0eae584b 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -25,17 +25,18 @@ class StatsController < ApplicationController # Genera un gráfico de visitas por dominio asociado a este sitio def host - - @stats = Rollup.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series| - series.each do |serie| - serie[:name] = serie.dig(:dimensions, 'host') - serie[:data].transform_values! do |value| - value * nodes + if stale? [last_stat, hostnames, interval] + stats = Rollup.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series| + series.each do |serie| + serie[:name] = serie.dig(:dimensions, 'host') + serie[:data].transform_values! do |value| + value * nodes + end end end end - render json: @stats + render json: stats end private From 224ea1ebc54ef6621614474eeaedd33f5dc633e3 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 8 Oct 2021 18:39:55 -0300 Subject: [PATCH 15/52] =?UTF-8?q?mostrar=20gr=C3=A1ficos=20de=20recursos?= =?UTF-8?q?=20utilizados?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 25 +++++++++++++++++++++++-- app/views/stats/index.haml | 11 +++++++++-- config/locales/en.yml | 18 +++++++++++++++--- config/locales/es.yml | 18 +++++++++++++++--- 4 files changed, 62 insertions(+), 10 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 0eae584b..d41573e2 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -9,6 +9,7 @@ class StatsController < ApplicationController before_action :authorize_stats INTERVALS = %i[hour day week month year].freeze + RESOURCES = %i[builds space_used build_time].freeze # XXX: Permitir a Chart.js inyectar su propio CSS content_security_policy only: :index do |policy| @@ -34,9 +35,22 @@ class StatsController < ApplicationController end end end - end - render json: stats + render json: stats + end + end + + def resources + if stale? [last_stat, interval, resource] + options = { + interval: interval, + dimensions: { + deploy_id: @site.deploys.where(type: 'DeployLocal').pluck(:id).first + } + } + + render json: Rollup.series(resource, **options) + end end private @@ -86,6 +100,13 @@ class StatsController < ApplicationController end end + def resource + @resource ||= begin + r = params[:resource].to_sym + RESOURCES.include?(r) ? r : :builds + end + end + # Obtiene la cantidad de nodos de Sutty, para poder calcular la # cantidad de visitas. # diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 63aa76cd..0251955e 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -13,5 +13,12 @@ = link_to t(".#{interval}"), site_stats_path(interval: interval), class: "btn #{'btn-primary active' if params[:interval].to_sym == interval}" .mb-3 - %h2= t('.host', count: @hostnames.size) - = line_chart site_stats_host_path(@chart_params), locale: 'es' + %h2= t('.host.title', count: @hostnames.size) + %p.lead= t('.host.description') + = line_chart site_stats_host_path(@chart_params), **@chart_options + + - StatsController::RESOURCES.each do |resource| + .mb-3 + %h2= t(".resources.#{resource}.title") + %p.lead= t(".resources.#{resource}.description") + = line_chart site_stats_resources_path(resource: resource, **@chart_params), **@chart_options diff --git a/config/locales/en.yml b/config/locales/en.yml index 58e0e726..08313a03 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -261,9 +261,21 @@ en: month: 'Monthly' year: 'Yearly' host: - zero: 'Site visits' - one: 'Site visits' - other: 'Visits by site name' + title: + zero: 'Site visits' + one: 'Site visits' + other: 'Visits by site name' + description: 'Counts visited pages on your site, grouped by domain names in use.' + resources: + builds: + title: 'Site publication' + description: 'Times you published your site.' + space_used: + title: 'Server disk usage' + description: 'Average storage space used by your site.' + build_time: + title: 'Publication time' + description: 'Average time your site takes to build.' sites: donations: url: 'https://donaciones.sutty.nl/en/' diff --git a/config/locales/es.yml b/config/locales/es.yml index 8df68896..b1161287 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -266,9 +266,21 @@ es: month: 'Mensuales' year: 'Anuales' host: - zero: 'Visitas del sitio' - one: 'Visitas del sitio' - other: 'Visitas por nombre del sitio' + title: + zero: 'Visitas del sitio' + one: 'Visitas del sitio' + other: 'Visitas por nombre del sitio' + description: 'Cuenta la cantidad de páginas visitadas en tu sitio, dividida por los nombres de dominio en uso.' + resources: + builds: + title: 'Publicaciones del sitio' + description: 'Cantidad de veces que publicaste tu sitio.' + space_used: + title: 'Espacio utilizado en el servidor' + description: 'Espacio en disco que ocupa en promedio tu sitio.' + build_time: + title: 'Tiempo de publicación' + description: 'Tiempo promedio que toma en publicarse tu sitio.' sites: donations: url: 'https://donaciones.sutty.nl/' From 833213ec80c343379ccdd566aaa86e92e051b9ad Mon Sep 17 00:00:00 2001 From: f Date: Fri, 8 Oct 2021 18:40:16 -0300 Subject: [PATCH 16/52] agregar opciones a cada recurso --- app/controllers/stats_controller.rb | 6 ++++++ app/views/stats/index.haml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index d41573e2..8045a653 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -10,6 +10,12 @@ class StatsController < ApplicationController INTERVALS = %i[hour day week month year].freeze RESOURCES = %i[builds space_used build_time].freeze + EXTRA_OPTIONS = { + builds: {}, + space_used: { bytes: true }, + build_time: {} + }.freeze + # XXX: Permitir a Chart.js inyectar su propio CSS content_security_policy only: :index do |policy| diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 0251955e..101dee1e 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -21,4 +21,4 @@ .mb-3 %h2= t(".resources.#{resource}.title") %p.lead= t(".resources.#{resource}.description") - = line_chart site_stats_resources_path(resource: resource, **@chart_params), **@chart_options + = line_chart site_stats_resources_path(resource: resource, **@chart_params), **@chart_options.merge(StatsController::EXTRA_OPTIONS[resource]) From 86f1ac450469cfce2d53be41b7f9d594e2d3a985 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 8 Oct 2021 18:40:52 -0300 Subject: [PATCH 17/52] =?UTF-8?q?fixup!=20mostrar=20gr=C3=A1ficos=20de=20r?= =?UTF-8?q?ecursos=20utilizados?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/policies/site_stat_policy.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/policies/site_stat_policy.rb b/app/policies/site_stat_policy.rb index d11be95d..6c98c775 100644 --- a/app/policies/site_stat_policy.rb +++ b/app/policies/site_stat_policy.rb @@ -16,4 +16,8 @@ class SiteStatPolicy def host? index? end + + def resources? + index? + end end From 51c2fdf6d6899d5b931e2ac301007d33deb641b5 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 8 Oct 2021 18:41:07 -0300 Subject: [PATCH 18/52] =?UTF-8?q?fixup!=20fixup!=20mostrar=20gr=C3=A1ficos?= =?UTF-8?q?=20de=20recursos=20utilizados?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/routes.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/config/routes.rb b/config/routes.rb index 0af58080..b59266fb 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -76,5 +76,6 @@ Rails.application.routes.draw do resources :stats, only: [:index] get :'stats/host', to: 'stats#host' + get :'stats/resources', to: 'stats#resources' end end From 034245595587990cd73ca3a34e945d2127471c9d Mon Sep 17 00:00:00 2001 From: f Date: Fri, 8 Oct 2021 18:42:34 -0300 Subject: [PATCH 19/52] =?UTF-8?q?encontrar=20p=C3=A1ginas=20y=20distinguir?= =?UTF-8?q?=20si=20fueron=20cyborgs=20o=20no?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/access_log.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/models/access_log.rb b/app/models/access_log.rb index 0dceb7d7..3a066b33 100644 --- a/app/models/access_log.rb +++ b/app/models/access_log.rb @@ -6,4 +6,7 @@ class AccessLog < ApplicationRecord # # @see {https://en.wikipedia.org/wiki/List_of_HTTP_status_codes} scope :completed_requests, -> { where(request_method: 'GET', request_completion: 'OK', status: [200, 304]) } + scope :non_robots, -> { where(crawler: false) } + scope :robots, -> { where(crawler: true) } + scope :pages, -> { where(sent_http_content_type: ['text/html', 'text/html; charset=utf-8', 'text/html; charset=UTF-8']) } end From 306d1ed983f69456a219a6c5fae47ea5e98ff8b1 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 9 Oct 2021 14:45:34 -0300 Subject: [PATCH 20/52] =?UTF-8?q?recolectar=20estad=C3=ADsticas=20una=20ve?= =?UTF-8?q?z=20por=20hora?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 66 +++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 app/jobs/stat_collection_job.rb diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb new file mode 100644 index 00000000..c36d036c --- /dev/null +++ b/app/jobs/stat_collection_job.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +# Genera resúmenes de información para poder mostrar estadísticas y se +# corre regularmente a sí misma. +class StatCollectionJob < ApplicationJob + class CrontabException < StandardError; end + + # Descartar y notificar si pasó algo más. + # + # XXX: En realidad deberíamos seguir reintentando? + discard_on(Exception) do |_, error| + ExceptionNotifier.notify_exception error + end + + # Correr indefinidamente una vez por hora. + # + # XXX: El orden importa, si el descarte viene después, nunca se va a + # reintentar. + retry_on(StatCollectionJob::CrontabException, wait: 1.hour, attempts: Float::INFINITY) + + COLUMNS = %i[uri].freeze + + def perform(once: false) + Stat::INTERVALS.each do |interval| + options = { interval: interval } + + # Visitas por hostname + AccessLog.completed_requests.group(:host).rollup('host', **options) + + combined_columns **options + stats_by_site **options + end + + # Registrar que se hicieron todas las recolecciones + Stat.create! + + raise CrontabException unless once + end + + private + + # Combinación de columnas + def combined_columns(**options) + COLUMNS.each do |column| + AccessLog.completed_requests.group(:host, column).rollup("hostname|#{column}", **options) + end + end + + # Uso de recursos por cada sitio. + # + # XXX: En realidad se agrupan por el deploy_id, que siempre será el + # del DeployLocal. + def stats_by_site(**options) + Site.find_each do |site| + site.build_stats.jekyll.group(:deploy_id).rollup('builds', **options) + + site.build_stats.jekyll.group(:deploy_id).rollup('space_used', **options) do |rollup| + rollup.average(:bytes) + end + + site.build_stats.jekyll.group(:deploy_id).rollup('build_time', **options) do |rollup| + rollup.average(:seconds) + end + end + end +end From e35cbe79eb53f3dceb082865360e060c827a237c Mon Sep 17 00:00:00 2001 From: f Date: Sat, 9 Oct 2021 14:46:58 -0300 Subject: [PATCH 21/52] =?UTF-8?q?recolectar=20visitas=20a=20p=C3=A1ginas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index c36d036c..04f62b8a 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -25,10 +25,10 @@ class StatCollectionJob < ApplicationJob options = { interval: interval } # Visitas por hostname - AccessLog.completed_requests.group(:host).rollup('host', **options) + AccessLog.completed_requests.non_robots.pages.group(:host).rollup('host', **options) - combined_columns **options - stats_by_site **options + combined_columns(**options) + stats_by_site(**options) end # Registrar que se hicieron todas las recolecciones @@ -42,7 +42,7 @@ class StatCollectionJob < ApplicationJob # Combinación de columnas def combined_columns(**options) COLUMNS.each do |column| - AccessLog.completed_requests.group(:host, column).rollup("hostname|#{column}", **options) + AccessLog.completed_requests.non_robots.pages.group(:host, column).rollup("host|#{column}", **options) end end From 11568e6ce8e8471077278590cbd7de5a67b6003d Mon Sep 17 00:00:00 2001 From: f Date: Sat, 9 Oct 2021 15:36:14 -0300 Subject: [PATCH 22/52] no distinguir entre todas las uris MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit para poder buscar descargas de archivos binarios después --- app/jobs/stat_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 04f62b8a..5b7da160 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -42,7 +42,7 @@ class StatCollectionJob < ApplicationJob # Combinación de columnas def combined_columns(**options) COLUMNS.each do |column| - AccessLog.completed_requests.non_robots.pages.group(:host, column).rollup("host|#{column}", **options) + AccessLog.completed_requests.non_robots.group(:host, column).rollup("host|#{column}", **options) end end From c927a56befb1128deb3e631774e1084361c93744 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 9 Oct 2021 15:47:42 -0300 Subject: [PATCH 23/52] ejecutar una vez por hora exacto --- app/jobs/stat_collection_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 5b7da160..6f8470b8 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -16,7 +16,7 @@ class StatCollectionJob < ApplicationJob # # XXX: El orden importa, si el descarte viene después, nunca se va a # reintentar. - retry_on(StatCollectionJob::CrontabException, wait: 1.hour, attempts: Float::INFINITY) + retry_on(StatCollectionJob::CrontabException, wait: 1.hour, attempts: Float::INFINITY, jitter: 0) COLUMNS = %i[uri].freeze From f36ea9629de280b7ef110d8d66d3f05097ce7520 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 9 Oct 2021 15:50:38 -0300 Subject: [PATCH 24/52] centralizar valores --- app/controllers/stats_controller.rb | 9 +++------ app/models/stat.rb | 5 ++++- app/views/stats/index.haml | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 8045a653..093fbe93 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -8,15 +8,12 @@ class StatsController < ApplicationController before_action :authenticate_usuarie! before_action :authorize_stats - INTERVALS = %i[hour day week month year].freeze - RESOURCES = %i[builds space_used build_time].freeze EXTRA_OPTIONS = { builds: {}, space_used: { bytes: true }, build_time: {} }.freeze - # XXX: Permitir a Chart.js inyectar su propio CSS content_security_policy only: :index do |policy| policy.style_src :self, :unsafe_inline @@ -85,7 +82,7 @@ class StatsController < ApplicationController # toma de estadísticas y restando el tiempo que pasó desde ese # momento. def chart_options - time = last_stat.created_at + (1.send(interval)) + time = last_stat.created_at + 1.try(interval) please_return_at = { please_return_at: distance_of_time_in_words(Time.now, time) } @chart_options ||= { @@ -102,14 +99,14 @@ class StatsController < ApplicationController def interval @interval ||= begin i = params[:interval].to_sym - INTERVALS.include?(i) ? i : :day + Stat::INTERVALS.include?(i) ? i : :day end end def resource @resource ||= begin r = params[:resource].to_sym - RESOURCES.include?(r) ? r : :builds + Stat::RESOURCES.include?(r) ? r : :builds end end diff --git a/app/models/stat.rb b/app/models/stat.rb index cf717b05..1e3af9e9 100644 --- a/app/models/stat.rb +++ b/app/models/stat.rb @@ -1,3 +1,6 @@ # frozen_string_literal: true -class Stat < ApplicationRecord; end +class Stat < ApplicationRecord + INTERVALS = %i[hour day week month year].freeze + RESOURCES = %i[builds space_used build_time].freeze +end diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 101dee1e..992200a4 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -9,7 +9,7 @@ "#{time_ago_in_words @last_stat.created_at}." .mb-3 - - StatsController::INTERVALS.each do |interval| + - Stat::INTERVALS.each do |interval| = link_to t(".#{interval}"), site_stats_path(interval: interval), class: "btn #{'btn-primary active' if params[:interval].to_sym == interval}" .mb-3 @@ -17,7 +17,7 @@ %p.lead= t('.host.description') = line_chart site_stats_host_path(@chart_params), **@chart_options - - StatsController::RESOURCES.each do |resource| + - Stat::RESOURCES.each do |resource| .mb-3 %h2= t(".resources.#{resource}.title") %p.lead= t(".resources.#{resource}.description") From 5ca8e3c923e88bc696f5644f1053a50ddd345e0d Mon Sep 17 00:00:00 2001 From: f Date: Sat, 9 Oct 2021 15:52:18 -0300 Subject: [PATCH 25/52] usar guards --- app/controllers/stats_controller.rb | 36 ++++++++++++++--------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 093fbe93..1dc468bb 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -29,31 +29,31 @@ class StatsController < ApplicationController # Genera un gráfico de visitas por dominio asociado a este sitio def host - if stale? [last_stat, hostnames, interval] - stats = Rollup.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series| - series.each do |serie| - serie[:name] = serie.dig(:dimensions, 'host') - serie[:data].transform_values! do |value| - value * nodes - end + return unless stale? [last_stat, hostnames, interval] + + stats = Rollup.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series| + series.each do |serie| + serie[:name] = serie.dig(:dimensions, 'host') + serie[:data].transform_values! do |value| + value * nodes end end - - render json: stats end + + render json: stats end def resources - if stale? [last_stat, interval, resource] - options = { - interval: interval, - dimensions: { - deploy_id: @site.deploys.where(type: 'DeployLocal').pluck(:id).first - } - } + return unless stale? [last_stat, interval, resource] - render json: Rollup.series(resource, **options) - end + options = { + interval: interval, + dimensions: { + deploy_id: @site.deploys.where(type: 'DeployLocal').pluck(:id).first + } + } + + render json: Rollup.series(resource, **options) end private From 6c485e5e18abf29d3b3cea0de584503417006c6e Mon Sep 17 00:00:00 2001 From: f Date: Sat, 9 Oct 2021 17:27:45 -0300 Subject: [PATCH 26/52] =?UTF-8?q?obtener=20estad=C3=ADsticas=20de=20cualqu?= =?UTF-8?q?ier=20link?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 39 +++++++++++++++++++++++++++++ app/policies/site_stat_policy.rb | 4 +++ app/views/stats/index.haml | 18 ++++++++++++- config/locales/en.yml | 6 +++++ config/locales/es.yml | 10 +++++--- config/routes.rb | 1 + 6 files changed, 73 insertions(+), 5 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 1dc468bb..ca834ebd 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -25,6 +25,7 @@ class StatsController < ApplicationController hostnames last_stat chart_options + normalized_urls end # Genera un gráfico de visitas por dominio asociado a este sitio @@ -56,6 +57,22 @@ class StatsController < ApplicationController render json: Rollup.series(resource, **options) end + def uris + return unless stale? [last_stat, hostnames, interval, normalized_urls] + + options = { host: hostnames, uri: normalized_paths } + stats = Rollup.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series| + series.each do |serie| + serie[:name] = serie.dig(:dimensions).slice('host', 'uri').values.join.sub('/index.html', '/') + serie[:data].transform_values! do |value| + value * nodes + end + end + end + + render json: stats + end + private def last_stat @@ -72,6 +89,28 @@ class StatsController < ApplicationController @hostnames ||= [@site.hostname, @site.alternative_hostnames].flatten end + # Normalizar las URLs + # + # @return [Array] + def normalized_urls + @normalized_urls ||= params.permit(:urls).try(:[], + :urls)&.split("\n")&.map(&:strip)&.select(&:present?)&.select do |uri| + uri.start_with? 'https://' + end&.map do |u| + # XXX: Eliminar + # @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1} + next u unless u.end_with? '/' + + "#{u}index.html" + end&.uniq || [] + end + + def normalized_paths + @normalized_paths ||= normalized_urls.map do |u| + "/#{u.split('/', 4).last}" + end + end + # Opciones por defecto para los gráficos. # # La invitación a volver dentro de X tiempo es para dar un estimado de diff --git a/app/policies/site_stat_policy.rb b/app/policies/site_stat_policy.rb index 6c98c775..cb62b507 100644 --- a/app/policies/site_stat_policy.rb +++ b/app/policies/site_stat_policy.rb @@ -20,4 +20,8 @@ class SiteStatPolicy def resources? index? end + + def uris? + index? + end end diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 992200a4..1ce61a98 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -10,13 +10,29 @@ .mb-3 - Stat::INTERVALS.each do |interval| - = link_to t(".#{interval}"), site_stats_path(interval: interval), class: "btn #{'btn-primary active' if params[:interval].to_sym == interval}" + = link_to t(".#{interval}"), site_stats_path(interval: interval, urls: params[:urls]), class: "btn #{'btn-primary active' if params[:interval].to_sym == interval}" .mb-3 %h2= t('.host.title', count: @hostnames.size) %p.lead= t('.host.description') = line_chart site_stats_host_path(@chart_params), **@chart_options + .mb-3 + - original_urls = params[:urls]&.split("\n")&.map(&:strip) + - rows = original_urls.size.zero? ? 3 : original_urls.size + %h2= t('.urls.title') + %p.lead= t('.urls.description') + %form + %input{ type: 'hidden', name: 'interval', value: @interval } + .form-group + %label{ for: 'urls' }= t('.urls.label') + %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: rows, aria_describedby: 'help-urls' }= params[:urls] + %small#help-urls.feedback.form-text.text-muted= t('.urls.help') + .form-group + %button.btn{ type: 'submit' }= t('.urls.submit') + - if @normalized_urls.present? + = line_chart site_stats_uris_path(urls: params[:urls], **@chart_params), **@chart_options + - Stat::RESOURCES.each do |resource| .mb-3 %h2= t(".resources.#{resource}.title") diff --git a/config/locales/en.yml b/config/locales/en.yml index 08313a03..af864e7b 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -266,6 +266,12 @@ en: one: 'Site visits' other: 'Visits by site name' description: 'Counts visited pages on your site, grouped by domain names in use.' + urls: + title: 'Visits by URL' + description: 'Counts visits or downloads on any URL.' + label: 'URLs ("links")' + help: 'Copy and paste a single URL per line' + submit: 'Get stats' resources: builds: title: 'Site publication' diff --git a/config/locales/es.yml b/config/locales/es.yml index b1161287..7d0fe2bb 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -271,6 +271,12 @@ es: one: 'Visitas del sitio' other: 'Visitas por nombre del sitio' description: 'Cuenta la cantidad de páginas visitadas en tu sitio, dividida por los nombres de dominio en uso.' + urls: + title: 'Visitas por dirección' + description: 'Cantidad de visitas o descargas por dirección.' + label: 'Direcciones web ("links", vínculos)' + help: 'Copia y pega una dirección por línea.' + submit: 'Obtener estadísticas' resources: builds: title: 'Publicaciones del sitio' @@ -605,7 +611,3 @@ es: edit: 'Editando' usuaries: index: 'Usuaries' - day: 'Día' - week: 'Semana' - month: 'Mes' - year: 'Año' diff --git a/config/routes.rb b/config/routes.rb index b59266fb..1b0f9e0b 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -76,6 +76,7 @@ Rails.application.routes.draw do resources :stats, only: [:index] get :'stats/host', to: 'stats#host' + get :'stats/uris', to: 'stats#uris' get :'stats/resources', to: 'stats#resources' end end From 7ab2ec5933613ce23cff5ab17e9141e0cf65c7ac Mon Sep 17 00:00:00 2001 From: f Date: Sat, 9 Oct 2021 17:28:26 -0300 Subject: [PATCH 27/52] =?UTF-8?q?no=20fallar=20si=20el=20intervalo=20est?= =?UTF-8?q?=C3=A1=20vac=C3=ADo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 2 +- app/views/stats/index.haml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index ca834ebd..a5ca04e1 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -137,7 +137,7 @@ class StatsController < ApplicationController # @return [Symbol] def interval @interval ||= begin - i = params[:interval].to_sym + i = params[:interval]&.to_sym Stat::INTERVALS.include?(i) ? i : :day end end diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 1ce61a98..d437aac1 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -10,7 +10,7 @@ .mb-3 - Stat::INTERVALS.each do |interval| - = link_to t(".#{interval}"), site_stats_path(interval: interval, urls: params[:urls]), class: "btn #{'btn-primary active' if params[:interval].to_sym == interval}" + = link_to t(".#{interval}"), site_stats_path(interval: interval, urls: params[:urls]), class: "btn #{'btn-primary active' if @interval == interval}" .mb-3 %h2= t('.host.title', count: @hostnames.size) From 5dad13bc3c4ff384ae44dc82ea9c9d5435d49832 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 9 Oct 2021 17:28:56 -0300 Subject: [PATCH 28/52] =?UTF-8?q?m=C3=A1s=20espacio=20entre=20los=20gr?= =?UTF-8?q?=C3=A1ficos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/stats/index.haml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index d437aac1..e4ef5f61 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -8,16 +8,16 @@ %time{ datetime: @last_stat.created_at } "#{time_ago_in_words @last_stat.created_at}." - .mb-3 + .mb-5 - Stat::INTERVALS.each do |interval| = link_to t(".#{interval}"), site_stats_path(interval: interval, urls: params[:urls]), class: "btn #{'btn-primary active' if @interval == interval}" - .mb-3 + .mb-5 %h2= t('.host.title', count: @hostnames.size) %p.lead= t('.host.description') = line_chart site_stats_host_path(@chart_params), **@chart_options - .mb-3 + .mb-5 - original_urls = params[:urls]&.split("\n")&.map(&:strip) - rows = original_urls.size.zero? ? 3 : original_urls.size %h2= t('.urls.title') @@ -34,7 +34,7 @@ = line_chart site_stats_uris_path(urls: params[:urls], **@chart_params), **@chart_options - Stat::RESOURCES.each do |resource| - .mb-3 + .mb-5 %h2= t(".resources.#{resource}.title") %p.lead= t(".resources.#{resource}.description") = line_chart site_stats_resources_path(resource: resource, **@chart_params), **@chart_options.merge(StatsController::EXTRA_OPTIONS[resource]) From fd2e2509d0e5f681c8866cf1c4eecdd51aac2275 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 9 Oct 2021 17:29:09 -0300 Subject: [PATCH 29/52] sin comillas --- app/views/stats/index.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index e4ef5f61..4119f8ae 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -6,7 +6,7 @@ %small = t('.last_update') %time{ datetime: @last_stat.created_at } - "#{time_ago_in_words @last_stat.created_at}." + #{time_ago_in_words @last_stat.created_at}. .mb-5 - Stat::INTERVALS.each do |interval| From 810f71e9c1348825303b990299ed7ad03e767bd7 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 9 Oct 2021 18:25:43 -0300 Subject: [PATCH 30/52] =?UTF-8?q?contar=20por=20qu=C3=A9=20mostramos=20los?= =?UTF-8?q?=20recursos=20usados?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/stats/index.haml | 6 +++++- config/locales/en.yml | 6 ++++-- config/locales/es.yml | 10 ++++++---- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index 4119f8ae..e69c2546 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -33,8 +33,12 @@ - if @normalized_urls.present? = line_chart site_stats_uris_path(urls: params[:urls], **@chart_params), **@chart_options + .mb-5 + %h2= t('.resources.title') + %p.lead= t('.resources.description') + - Stat::RESOURCES.each do |resource| .mb-5 - %h2= t(".resources.#{resource}.title") + %h3= t(".resources.#{resource}.title") %p.lead= t(".resources.#{resource}.description") = line_chart site_stats_resources_path(resource: resource, **@chart_params), **@chart_options.merge(StatsController::EXTRA_OPTIONS[resource]) diff --git a/config/locales/en.yml b/config/locales/en.yml index af864e7b..be327035 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -264,15 +264,17 @@ en: title: zero: 'Site visits' one: 'Site visits' - other: 'Visits by site name' + other: 'Visits by domain name' description: 'Counts visited pages on your site, grouped by domain names in use.' urls: title: 'Visits by URL' description: 'Counts visits or downloads on any URL.' label: 'URLs ("links")' help: 'Copy and paste a single URL per line' - submit: 'Get stats' + submit: 'Update graph' resources: + title: 'Resource usage' + description: "In this section you can find statistics on your site's use of Sutty's shared resources" builds: title: 'Site publication' description: 'Times you published your site.' diff --git a/config/locales/es.yml b/config/locales/es.yml index 7d0fe2bb..774a8cfd 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -269,15 +269,17 @@ es: title: zero: 'Visitas del sitio' one: 'Visitas del sitio' - other: 'Visitas por nombre del sitio' - description: 'Cuenta la cantidad de páginas visitadas en tu sitio, dividida por los nombres de dominio en uso.' + other: 'Visitas agrupadas por nombre de dominio del sitio' + description: 'Cuenta la cantidad de páginas visitadas en tu sitio.' urls: title: 'Visitas por dirección' description: 'Cantidad de visitas o descargas por dirección.' - label: 'Direcciones web ("links", vínculos)' + label: 'Direcciones web (URL, "links", vínculos)' help: 'Copia y pega una dirección por línea.' - submit: 'Obtener estadísticas' + submit: 'Actualizar gráfico' resources: + title: 'Uso de recursos' + description: 'En esta sección podrás acceder a estadísticas del uso de recursos compartidos con otros sitios alojados en Sutty.' builds: title: 'Publicaciones del sitio' description: 'Cantidad de veces que publicaste tu sitio.' From ec775847b9dd8b446ee2947826452c6b5782d3d1 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 9 Oct 2021 18:27:42 -0300 Subject: [PATCH 31/52] sugerir algunas URLs --- app/controllers/stats_controller.rb | 2 +- app/views/stats/index.haml | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index a5ca04e1..3606f37e 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -102,7 +102,7 @@ class StatsController < ApplicationController next u unless u.end_with? '/' "#{u}index.html" - end&.uniq || [] + end&.uniq || [@site.url, @site.urls].flatten.uniq end def normalized_paths diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index e69c2546..fb5c0d52 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -18,15 +18,13 @@ = line_chart site_stats_host_path(@chart_params), **@chart_options .mb-5 - - original_urls = params[:urls]&.split("\n")&.map(&:strip) - - rows = original_urls.size.zero? ? 3 : original_urls.size %h2= t('.urls.title') %p.lead= t('.urls.description') %form %input{ type: 'hidden', name: 'interval', value: @interval } .form-group %label{ for: 'urls' }= t('.urls.label') - %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: rows, aria_describedby: 'help-urls' }= params[:urls] + %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size, aria_describedby: 'help-urls' }= @normalized_urls.join("\n") %small#help-urls.feedback.form-text.text-muted= t('.urls.help') .form-group %button.btn{ type: 'submit' }= t('.urls.submit') From 356db8546522ac581254f11d2dcdd4419dfc6595 Mon Sep 17 00:00:00 2001 From: f Date: Sat, 9 Oct 2021 18:28:38 -0300 Subject: [PATCH 32/52] =?UTF-8?q?empezar=20a=20recolectar=20estad=C3=ADsti?= =?UTF-8?q?cas=20cuando=20se=20inicia=20el=20panel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/application.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/application.rb b/config/application.rb index 7326ae0f..4922b74f 100644 --- a/config/application.rb +++ b/config/application.rb @@ -48,6 +48,11 @@ module Sutty EmailAddress::Config.error_messages translations.transform_keys(&:to_s), locale.to_s end + + # Empezar el recolector de datos en la próxima hora + now = Time.now + next_hour = now.at_beginning_of_hour + 1.hour + 1.minute + StatCollectionJob.perform_in(next_hour - now, once: false) end end end From aa86ead1124a6cfd865956d2a4d5bd8ace4f81de Mon Sep 17 00:00:00 2001 From: f Date: Sat, 9 Oct 2021 18:29:53 -0300 Subject: [PATCH 33/52] ruboyuta --- app/controllers/stats_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 3606f37e..3fe821ac 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -63,7 +63,7 @@ class StatsController < ApplicationController options = { host: hostnames, uri: normalized_paths } stats = Rollup.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series| series.each do |serie| - serie[:name] = serie.dig(:dimensions).slice('host', 'uri').values.join.sub('/index.html', '/') + serie[:name] = serie[:dimensions].slice('host', 'uri').values.join.sub('/index.html', '/') serie[:data].transform_values! do |value| value * nodes end From d06edc2f62ce1e2e499684e3446d1a39c90d81f3 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 20 Oct 2021 12:52:31 -0300 Subject: [PATCH 34/52] usar rollups desde git para poder hacer un rollup recursivo --- Gemfile | 2 +- Gemfile.lock | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Gemfile b/Gemfile index b5e8664b..a04353b2 100644 --- a/Gemfile +++ b/Gemfile @@ -59,7 +59,7 @@ gem 'rails-i18n' gem 'rails_warden' gem 'redis', require: %w[redis redis/connection/hiredis] gem 'redis-rails' -gem 'rollups' +gem 'rollups', git: 'https://github.com/ankane/rollup.git', branch: 'master' gem 'rubyzip' gem 'rugged' gem 'concurrent-ruby-ext' diff --git a/Gemfile.lock b/Gemfile.lock index 9d1f512b..c1f6925c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,6 +6,15 @@ GIT rails (>= 3.0) rake (>= 0.8.7) +GIT + remote: https://github.com/ankane/rollup.git + revision: 94ca777d54180c23e96ac4b4285cc9b405ccbd1a + branch: master + specs: + rollups (0.1.2) + activesupport (>= 5.1) + groupdate (>= 5.2) + GIT remote: https://github.com/fauno/email_address revision: 536b51f7071b68a55140c0c1726b4cd401d1c04d @@ -483,9 +492,6 @@ GEM actionpack (>= 5.0) railties (>= 5.0) rexml (3.2.5) - rollups (0.1.2) - activesupport (>= 5.1) - groupdate (>= 5.2) rouge (3.26.0) rubocop (1.18.3) parallel (~> 1.10) @@ -700,7 +706,7 @@ DEPENDENCIES recursero-jekyll-theme redis redis-rails - rollups + rollups! rubocop-rails rubyzip rugged From 245973b5196d478e64d7f3f14ae73fda72cd0e82 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 20 Oct 2021 12:53:42 -0300 Subject: [PATCH 35/52] =?UTF-8?q?no=20fallar=20si=20todav=C3=ADa=20no=20ha?= =?UTF-8?q?y=20stats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/stats/index.haml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index fb5c0d52..bfcf33ef 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -2,11 +2,12 @@ .col %h1= t('.title') %p.lead= t('.help') - %p - %small - = t('.last_update') - %time{ datetime: @last_stat.created_at } - #{time_ago_in_words @last_stat.created_at}. + - if @last_stat + %p + %small + = t('.last_update') + %time{ datetime: @last_stat.created_at } + #{time_ago_in_words @last_stat.created_at}. .mb-5 - Stat::INTERVALS.each do |interval| From cc3535097e4924ed4d1ba87da04be769ba9b1b9e Mon Sep 17 00:00:00 2001 From: f Date: Wed, 20 Oct 2021 12:57:11 -0300 Subject: [PATCH 36/52] =?UTF-8?q?todav=C3=ADa=20no=20empezar=20la=20recole?= =?UTF-8?q?cci=C3=B3n=20autom=C3=A1tica?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/application.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/config/application.rb b/config/application.rb index 4922b74f..7326ae0f 100644 --- a/config/application.rb +++ b/config/application.rb @@ -48,11 +48,6 @@ module Sutty EmailAddress::Config.error_messages translations.transform_keys(&:to_s), locale.to_s end - - # Empezar el recolector de datos en la próxima hora - now = Time.now - next_hour = now.at_beginning_of_hour + 1.hour + 1.minute - StatCollectionJob.perform_in(next_hour - now, once: false) end end end From 1497113f73a8f3cd5cb243b41236445f1667dfdc Mon Sep 17 00:00:00 2001 From: f Date: Wed, 20 Oct 2021 12:58:19 -0300 Subject: [PATCH 37/52] usar menos intervalos las horas y semanas generan demasiados rollups --- app/models/stat.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/stat.rb b/app/models/stat.rb index 1e3af9e9..c986ba4b 100644 --- a/app/models/stat.rb +++ b/app/models/stat.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true class Stat < ApplicationRecord - INTERVALS = %i[hour day week month year].freeze + INTERVALS = %i[year month day].freeze RESOURCES = %i[builds space_used build_time].freeze end From 849ee4491c0579133f8134719510f079cff10b66 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 20 Oct 2021 12:58:51 -0300 Subject: [PATCH 38/52] =?UTF-8?q?instalar=20las=20librer=C3=ADas=20en=20pr?= =?UTF-8?q?oducci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 184f7ebb..d520c8f5 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,8 @@ "@rails/ujs": "^6.1.3-1", "@rails/webpacker": "5.2.1", "babel-loader": "^8.2.2", + "chart.js": "^3.5.1", + "chartkick": "^4.0.5", "circular-dependency-plugin": "^5.2.2", "commonmark": "^0.29.0", "fork-awesome": "^1.1.7", @@ -30,8 +32,6 @@ "zepto": "^1.2.0" }, "devDependencies": { - "@types/rails__activestorage": "^6.0.0", - "chart.js": "^3.5.1", - "chartkick": "^4.0.5" + "@types/rails__activestorage": "^6.0.0" } } From 9cf7c6286169ee4f5ee1cac1ea4f3f1ad8106a3d Mon Sep 17 00:00:00 2001 From: f Date: Wed, 20 Oct 2021 13:08:21 -0300 Subject: [PATCH 39/52] procesar uris a demanda --- app/jobs/uri_collection_job.rb | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 app/jobs/uri_collection_job.rb diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb new file mode 100644 index 00000000..79b83644 --- /dev/null +++ b/app/jobs/uri_collection_job.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +# Procesar una lista de URIs para una lista de dominios. Esto nos +# permite procesar estadísticas a demanada. +class UriCollectionJob < ApplicationJob + def perform(hostnames:, file:) + uris = File.read(file).split("\n") + + hostnames.each do |hostname| + uris.each do |uri| + break if File.exist? Rails.root.join('tmp', 'uri_collection_job_stop') + + AccessLog.where(host: hostname, uri: uri).completed_requests.non_robots.group(:host, :uri).rollup('host|uri', interval: 'day') + end + end + end +end From 8d38d0d2aedca550e5d8815f6d4b45c70138cfdc Mon Sep 17 00:00:00 2001 From: f Date: Wed, 20 Oct 2021 13:14:19 -0300 Subject: [PATCH 40/52] =?UTF-8?q?recolecci=C3=B3n=20de=20estad=C3=ADsticas?= =?UTF-8?q?=20con=20mejor=20performance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/stat_collection_job.rb | 42 ++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 6f8470b8..a49a1635 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -21,14 +21,20 @@ class StatCollectionJob < ApplicationJob COLUMNS = %i[uri].freeze def perform(once: false) - Stat::INTERVALS.each do |interval| - options = { interval: interval } + Site.find_each do |site| + hostnames = [site.hostname, site.alternative_hostnames].flatten + + # Usamos el primero porque luego podemos hacer un rollup recursivo + options = { interval: Stat::INTERVALS.first } # Visitas por hostname - AccessLog.completed_requests.non_robots.pages.group(:host).rollup('host', **options) + hostnames.each do |hostname| + AccessLog.where(host: hostname).completed_requests.non_robots.pages.group(:host).rollup('host', **options) - combined_columns(**options) - stats_by_site(**options) + combined_columns(hostname, **options) + end + + stats_by_site(site, **options) end # Registrar que se hicieron todas las recolecciones @@ -40,9 +46,15 @@ class StatCollectionJob < ApplicationJob private # Combinación de columnas - def combined_columns(**options) + def combined_columns(hostname, **options) + where = { host: hostname } + COLUMNS.each do |column| - AccessLog.completed_requests.non_robots.group(:host, column).rollup("host|#{column}", **options) + AccessLog.where(host: hostname).pluck(Arel.sql("distinct #{column}")).each do |value| + where[column] = value + + AccessLog.where(**where).completed_requests.non_robots.group(:host, column).rollup("host|#{column}", **options) + end end end @@ -50,17 +62,15 @@ class StatCollectionJob < ApplicationJob # # XXX: En realidad se agrupan por el deploy_id, que siempre será el # del DeployLocal. - def stats_by_site(**options) - Site.find_each do |site| - site.build_stats.jekyll.group(:deploy_id).rollup('builds', **options) + def stats_by_site(site, **options) + site.build_stats.jekyll.group(:deploy_id).rollup('builds', **options) - site.build_stats.jekyll.group(:deploy_id).rollup('space_used', **options) do |rollup| - rollup.average(:bytes) - end + site.build_stats.jekyll.group(:deploy_id).rollup('space_used', **options) do |rollup| + rollup.average(:bytes) + end - site.build_stats.jekyll.group(:deploy_id).rollup('build_time', **options) do |rollup| - rollup.average(:seconds) - end + site.build_stats.jekyll.group(:deploy_id).rollup('build_time', **options) do |rollup| + rollup.average(:seconds) end end end From 7e1696204183e73afa307e27d77ecbfd87f7c5b8 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 20 Oct 2021 17:36:30 -0300 Subject: [PATCH 41/52] no fallar si no hay stat closes #3228 --- app/controllers/stats_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 3fe821ac..b9b93e9f 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -121,7 +121,7 @@ class StatsController < ApplicationController # toma de estadísticas y restando el tiempo que pasó desde ese # momento. def chart_options - time = last_stat.created_at + 1.try(interval) + time = (last_stat&.created_at || Time.now) + 1.try(interval) please_return_at = { please_return_at: distance_of_time_in_words(Time.now, time) } @chart_options ||= { From 7227ecfe2a1bfb99c19e5af8998b2253c323f348 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 21 Oct 2021 20:53:25 -0300 Subject: [PATCH 42/52] solo instalar las gemas de assets en desarrollo --- .env.example | 1 + Gemfile | 17 ++++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index a62e2b0a..f79ff3a4 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,4 @@ +RAILS_GROUPS=assets DELEGATE=athshe.sutty.nl HAINISH=../haini.sh/haini.sh DATABASE= diff --git a/Gemfile b/Gemfile index db06827d..01167fa1 100644 --- a/Gemfile +++ b/Gemfile @@ -11,13 +11,16 @@ gem 'dotenv-rails', require: 'dotenv/rails-now' gem 'rails', '~> 6' # Use Puma as the app server gem 'puma' -# See https://github.com/rails/execjs#readme for more supported runtimes -# gem 'therubyracer', platforms: :ruby -# Use SCSS for stylesheets -gem 'sassc-rails' -# Use Uglifier as compressor for JavaScript assets -gem 'uglifier', '>= 1.3.0' -gem 'bootstrap', '~> 4' + +# Solo incluir las gemas cuando estemos en desarrollo o compilando los +# assets. No es necesario instalarlas en producción. +# +# XXX: Supuestamente Rails ya soporta RAILS_GROUPS, pero Bundler no. +if ENV['RAILS_GROUPS']&.include? 'assets' + gem 'sassc-rails' + gem 'uglifier', '>= 1.3.0' + gem 'bootstrap', '~> 4' +end # Turbolinks makes navigating your web application faster. Read more: # https://github.com/turbolinks/turbolinks From fd784919df49d8703713b0f741521079de37fdc4 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 21 Oct 2021 20:54:12 -0300 Subject: [PATCH 43/52] rails ya corre yarn antes de compilar --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 36139d7a..5aca94f8 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ help: always ## Ayuda @echo -e "\nArgumentos:\n" @grep -E "^[a-z\-]+ \?=.*##" Makefile | sed -re "s/(.*) \?=.*##(.*)/\1;\2/" | column -s ";" -t | sed -re "s/^([^ ]+) /\x1B[38;5;197m\1\x1B[0m/" -assets: node_modules public/packs/manifest.json.br ## Compilar los assets +assets: public/packs/manifest.json.br ## Compilar los assets test: always ## Ejecutar los tests $(MAKE) rake args="test RAILS_ENV=test $(args)" From 084bf8547f0202d5409d1c36a0cab71a40e7322f Mon Sep 17 00:00:00 2001 From: f Date: Thu, 21 Oct 2021 20:54:50 -0300 Subject: [PATCH 44/52] ya no es necesario hacer limpieza en docker --- Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0b3253b4..07322903 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,10 +60,6 @@ RUN mv ../sutty/.bundle ./.bundle # Instalar secretos COPY --chown=app:root ./config/credentials.yml.enc ./config/ -# Eliminar la necesidad de un runtime JS en producción, porque los -# assets ya están pre-compilados. -RUN sed -re "/(sassc|uglifier|bootstrap|coffee-rails)/d" -i Gemfile -RUN bundle clean RUN rm -rf ./node_modules ./tmp/cache ./.git ./test ./doc # Eliminar archivos innecesarios USER root From 570761fab5efe47f0bbdf389550424c726dacc12 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 21 Oct 2021 20:55:40 -0300 Subject: [PATCH 45/52] =?UTF-8?q?actualizar=20contenedor=20con=20una=20ver?= =?UTF-8?q?si=C3=B3n=20espec=C3=ADfica=20de=20bundler?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 07322903..ee6ba871 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # el mismo repositorio de trabajo. Cuando tengamos CI/CD algunas cosas # como el tarball van a tener que cambiar porque ya vamos a haber hecho # un clone/pull limpio. -FROM alpine:3.13.5 AS build +FROM alpine:3.13.6 AS build MAINTAINER "f " ARG RAILS_MASTER_KEY @@ -14,10 +14,10 @@ ENV SECRET_KEY_BASE solo_es_necesaria_para_correr_rake ENV RAILS_ENV production ENV RAILS_MASTER_KEY=$RAILS_MASTER_KEY -RUN apk add --no-cache libxslt libxml2 tzdata ruby ruby-bundler ruby-json ruby-bigdecimal ruby-rake +RUN apk add --no-cache libxslt libxml2 tzdata ruby ruby-json ruby-bigdecimal ruby-rake RUN apk add --no-cache postgresql-libs git yarn brotli libssh2 python3 -RUN test "2.7.3" = `ruby -e 'puts RUBY_VERSION'` +RUN test "2.7.4" = `ruby -e 'puts RUBY_VERSION'` # https://github.com/rubygems/rubygems/issues/2918 # https://gitlab.alpinelinux.org/alpine/aports/issues/10808 @@ -29,7 +29,7 @@ RUN cd /usr/lib/ruby/2.7.0 && patch -Np 0 -i /tmp/rubygems-platform-musl.patch RUN addgroup -g 82 -S www-data RUN adduser -s /bin/sh -G www-data -h /home/app -D app RUN install -dm750 -o app -g www-data /home/app/sutty -RUN gem install --no-document bundler +RUN gem install --no-document bundler:2.1.4 # Empezamos con la usuaria app USER app @@ -39,7 +39,8 @@ WORKDIR /home/app/sutty # Copiamos solo el Gemfile para poder instalar las gemas necesarias COPY --chown=app:www-data ./Gemfile . COPY --chown=app:www-data ./Gemfile.lock . -RUN bundle config set no-cache 'true' +RUN bundle config set no-cache true +RUN bundle config set specific_platform true RUN bundle install --path=./vendor --without='test development' # Vaciar la caché RUN rm vendor/ruby/2.7.0/cache/*.gem @@ -67,7 +68,7 @@ RUN apk add --no-cache findutils RUN find /home/app/checkout/vendor/ruby/2.7.0 -maxdepth 3 -type d -name test -o -name spec -o -name rubocop | xargs -r rm -rf # Contenedor final -FROM sutty/monit:latest +FROM registry.nulo.in/sutty/monit:3.13.6 ENV RAILS_ENV production # Pandoc @@ -75,13 +76,13 @@ RUN echo 'http://dl-cdn.alpinelinux.org/alpine/edge/testing' >> /etc/apk/reposit # Instalar las dependencias, separamos la librería de base de datos para # poder reutilizar este primer paso desde otros contenedores -RUN apk add --no-cache libxslt libxml2 tzdata ruby ruby-bundler ruby-json ruby-bigdecimal ruby-rake ruby-irb +RUN apk add --no-cache libxslt libxml2 tzdata ruby ruby-json ruby-bigdecimal ruby-rake ruby-irb ruby-io-console ruby-etc RUN apk add --no-cache postgresql-libs libssh2 file rsync git jpegoptim vips RUN apk add --no-cache ffmpeg imagemagick pandoc tectonic oxipng jemalloc RUN apk add --no-cache git-lfs openssh-client patch # Chequear que la versión de ruby sea la correcta -RUN test "2.7.3" = `ruby -e 'puts RUBY_VERSION'` +RUN test "2.7.4" = `ruby -e 'puts RUBY_VERSION'` # https://github.com/rubygems/rubygems/issues/2918 # https://gitlab.alpinelinux.org/alpine/aports/issues/10808 @@ -93,7 +94,7 @@ RUN apk add --no-cache patch && cd /usr/lib/ruby/2.7.0 && patch -Np 0 -i /tmp/ru # principal RUN apk add --no-cache yarn # Instalar foreman para poder correr los servicios -RUN gem install --no-document --no-user-install bundler foreman +RUN gem install --no-document --no-user-install bundler:2.1.4 foreman # Agregar el grupo del servidor web y la usuaria RUN addgroup -g 82 -S www-data From 84e543ac078abfd77ecb8662a791ae67ba1eda01 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 22 Oct 2021 17:13:26 -0300 Subject: [PATCH 46/52] ignorar los cambios de licencia en idiomas no soportados --- app/services/site_service.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 389549c3..5e2fc706 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -122,6 +122,8 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do # la búsqueda. def change_licencias site.locales.each do |locale| + next unless I18n.available_locales.include? locale + Mobility.with_locale(locale) do permalink = "#{I18n.t('activerecord.models.licencia').downcase}/" post = site.posts(lang: locale).find_by(permalink: permalink) From ab004fae700db1f3b72cd1a44ec403fdabb8c452 Mon Sep 17 00:00:00 2001 From: f Date: Fri, 22 Oct 2021 18:21:47 -0300 Subject: [PATCH 47/52] decodificar las urls para poder buscarlas en el log --- app/controllers/stats_controller.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index b9b93e9f..44073c1f 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -108,6 +108,8 @@ class StatsController < ApplicationController def normalized_paths @normalized_paths ||= normalized_urls.map do |u| "/#{u.split('/', 4).last}" + end.map do |u| + URI.decode_www_form_component u end end From c84462c4a8342ac9331718335c529d636ebb9298 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Oct 2021 11:33:15 -0300 Subject: [PATCH 48/52] =?UTF-8?q?recolectar=20estad=C3=ADsticas=20usando?= =?UTF-8?q?=20menos=20recursos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/periodic_job.rb | 55 +++++++++++++++++ app/jobs/stat_collection_job.rb | 93 +++++++++++++---------------- app/jobs/uri_collection_job.rb | 101 ++++++++++++++++++++++++++++++-- app/models/site.rb | 1 + app/models/stat.rb | 6 +- 5 files changed, 198 insertions(+), 58 deletions(-) create mode 100644 app/jobs/periodic_job.rb diff --git a/app/jobs/periodic_job.rb b/app/jobs/periodic_job.rb new file mode 100644 index 00000000..8d9453a3 --- /dev/null +++ b/app/jobs/periodic_job.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +# Una tarea que se corre periódicamente +class PeriodicJob < ApplicationJob + class RunAgainException < StandardError; end + + STARTING_INTERVAL = Stat::INTERVALS.first + + # Tener el sitio a mano + attr_reader :site + + # Descartar y notificar si pasó algo más. + # + # XXX: En realidad deberíamos seguir reintentando? + discard_on(StandardError) do |_, error| + ExceptionNotifier.notify_exception(error) + end + + # Correr indefinidamente una vez por hora. + # + # XXX: El orden importa, si el descarte viene después, nunca se va a + # reintentar. + retry_on(PeriodicJob::RunAgainException, wait: 1.try(STARTING_INTERVAL), attempts: Float::INFINITY, jitter: 0) + + private + + # Las clases que implementen esta tienen que usar este método al + # terminar. + def run_again! + raise PeriodicJob::RunAgainException, 'Reintentando' + end + + # El intervalo de inicio + # + # @return [Symbol] + def starting_interval + STARTING_INTERVAL + end + + # La última recolección de estadísticas o empezar desde el principio + # de los tiempos. + # + # @return [Stat] + def last_stat + @last_stat ||= site.stats.where(name: stat_name).last || + site.stats.build(created_at: Time.new(1970, 1, 1)) + end + + # Devuelve el comienzo del intervalo + # + # @return [Time] + def beginning_of_interval + @beginning_of_interval ||= last_stat.created_at.try(:"beginning_of_#{starting_interval}") + end +end diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index a49a1635..2aa8d702 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -3,74 +3,65 @@ # Genera resúmenes de información para poder mostrar estadísticas y se # corre regularmente a sí misma. class StatCollectionJob < ApplicationJob - class CrontabException < StandardError; end + STAT_NAME = 'stat_collection_job' - # Descartar y notificar si pasó algo más. - # - # XXX: En realidad deberíamos seguir reintentando? - discard_on(Exception) do |_, error| - ExceptionNotifier.notify_exception error - end + def perform(site_id:, once: true) + @site = Site.find site_id - # Correr indefinidamente una vez por hora. - # - # XXX: El orden importa, si el descarte viene después, nunca se va a - # reintentar. - retry_on(StatCollectionJob::CrontabException, wait: 1.hour, attempts: Float::INFINITY, jitter: 0) + scope.rollup('builds', **options) - COLUMNS = %i[uri].freeze + scope.rollup('space_used', **options) do |rollup| + rollup.average(:bytes) + end - def perform(once: false) - Site.find_each do |site| - hostnames = [site.hostname, site.alternative_hostnames].flatten + scope.rollup('build_time', **options) do |rollup| + rollup.average(:seconds) + end - # Usamos el primero porque luego podemos hacer un rollup recursivo - options = { interval: Stat::INTERVALS.first } + # XXX: Es correcto promediar promedios? + Stat::INTERVALS.reduce do |previous, current| + rollup(name: 'builds', interval_previous: previous, interval: current) + rollup(name: 'space_used', interval_previous: previous, interval: current, operation: :average) + rollup(name: 'build_time', interval_previous: previous, interval: current, operation: :average) - # Visitas por hostname - hostnames.each do |hostname| - AccessLog.where(host: hostname).completed_requests.non_robots.pages.group(:host).rollup('host', **options) - - combined_columns(hostname, **options) - end - - stats_by_site(site, **options) + current end # Registrar que se hicieron todas las recolecciones - Stat.create! + site.stats.create! name: STAT_NAME - raise CrontabException unless once + run_again! unless once end private - # Combinación de columnas - def combined_columns(hostname, **options) - where = { host: hostname } - - COLUMNS.each do |column| - AccessLog.where(host: hostname).pluck(Arel.sql("distinct #{column}")).each do |value| - where[column] = value - - AccessLog.where(**where).completed_requests.non_robots.group(:host, column).rollup("host|#{column}", **options) - end - end + # Genera un rollup recursivo en base al período anterior y aplica una + # operación. + # + # @return [NilClass] + def rollup(name:, interval_previous:, interval:, operation: :sum) + Rollup.where(name: name, interval: interval_previous) + .where_dimensions(site_id: site.id) + .group("dimensions->'site_id'") + .rollup(name, interval: interval, update: true) do |rollup| + rollup.try(:operation, :value) + end end - # Uso de recursos por cada sitio. + # Los registros a procesar # - # XXX: En realidad se agrupan por el deploy_id, que siempre será el - # del DeployLocal. - def stats_by_site(site, **options) - site.build_stats.jekyll.group(:deploy_id).rollup('builds', **options) + # @return [ActiveRecord::Relation] + def scope + @scope ||= site.build_stats + .jekyll + .where('created_at => ?', beginning_of_interval) + .group(:site_id) + end - site.build_stats.jekyll.group(:deploy_id).rollup('space_used', **options) do |rollup| - rollup.average(:bytes) - end - - site.build_stats.jekyll.group(:deploy_id).rollup('build_time', **options) do |rollup| - rollup.average(:seconds) - end + # Las opciones por defecto + # + # @return [Hash] + def options + @options ||= { interval: starting_interval, update: true } end end diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 79b83644..9ec333cd 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -1,16 +1,105 @@ # frozen_string_literal: true # Procesar una lista de URIs para una lista de dominios. Esto nos -# permite procesar estadísticas a demanada. -class UriCollectionJob < ApplicationJob - def perform(hostnames:, file:) - uris = File.read(file).split("\n") +# permite procesar estadísticas a demanda. +# +# Hay varias cosas acá que van a convertirse en métodos propios, como la +# detección de URIs de un sitio (aunque la versión actual detecta todas +# las páginas y no solo las de posts como tenemos planeado, hay que +# resolver eso). +# +# Los hostnames de un sitio van a poder obtenerse a partir de +# Site#hostnames con la garantía de que son únicos. +class UriCollectionJob < PeriodicJob + # Ignoramos imágenes porque suelen ser demasiadas y no aportan a las + # estadísticas. + IMAGES = %w[.png .jpg .jpeg .gif .webp].freeze + STAT_NAME = 'uri_collection_job' + + def perform(site_id:, once: true) + @site = Site.find site_id hostnames.each do |hostname| uris.each do |uri| - break if File.exist? Rails.root.join('tmp', 'uri_collection_job_stop') + return if File.exist? Rails.root.join('tmp', 'uri_collection_job_stop') - AccessLog.where(host: hostname, uri: uri).completed_requests.non_robots.group(:host, :uri).rollup('host|uri', interval: 'day') + AccessLog.where(host: hostname, uri: uri) + .where('created_at >= ?', beginning_of_interval) + .completed_requests + .non_robots + .group(:host, :uri) + .rollup('host|uri', interval: starting_interval, update: true) + + # Reducir las estadísticas calculadas aplicando un rollup sobre el + # intervalo más amplio. + Stat::INTERVALS.reduce do |previous, current| + Rollup.where(name: 'host|uri', interval: previous) + .where_dimensions(host: hostname, uri: uri) + .group("dimensions->'host'", "dimensions->'uri'") + .rollup('host|uri', interval: current, update: true) do |rollup| + rollup.sum(:value) + end + + # Devolver el intervalo actual + current + end + end + end + + # Recordar la última vez que se corrió la tarea + site.stats.create! name: STAT_NAME + + run_again! unless once + end + + private + + def stat_name + STAT_NAME + end + + # @return [String] + # + # TODO: Cambiar al mergear origin-referer + def destination + @destination ||= site.deploys.find_by(type: 'DeployLocal').destination + end + + # TODO: Cambiar al mergear origin-referer + # + # @return [Array] + def hostnames + @hostnames ||= site.deploys.map do |deploy| + case deploy + when DeployLocal + site.hostname + when DeployWww + deploy.fqdn + when DeployAlternativeDomain + deploy.hostname.dup.tap do |h| + h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}") + end + when DeployHiddenService + deploy.onion + end + end.compact + end + + # Recolecta todas las URIs menos imágenes + # + # @return [Array] + def uris + @uris ||= Dir.chdir destination do + (Dir.glob('**/*.html') + Dir.glob('public/**/*').reject do |p| + File.directory? p + end.reject do |p| + p = p.downcase + + IMAGES.any? do |i| + p.end_with? i + end + end).map do |uri| + "/#{uri}" end end end diff --git a/app/models/site.rb b/app/models/site.rb index ddfe2bc9..df92264a 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -37,6 +37,7 @@ class Site < ApplicationRecord belongs_to :design belongs_to :licencia + has_many :stats has_many :log_entries, dependent: :destroy has_many :deploys, dependent: :destroy has_many :build_stats, through: :deploys diff --git a/app/models/stat.rb b/app/models/stat.rb index c986ba4b..5f72ccd0 100644 --- a/app/models/stat.rb +++ b/app/models/stat.rb @@ -1,6 +1,10 @@ # frozen_string_literal: true +# Registran cuándo fue la última recolección de datos. class Stat < ApplicationRecord - INTERVALS = %i[year month day].freeze + # XXX: Los intervalos van en orden de mayor especificidad a menor + INTERVALS = %i[day month year].freeze RESOURCES = %i[builds space_used build_time].freeze + + belongs_to :site end From 7a780188e4f85030d619014061e3ce09ef060b85 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Oct 2021 16:19:42 -0300 Subject: [PATCH 49/52] asegurarse que siempre buscamos assets --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 01167fa1..981ce7fd 100644 --- a/Gemfile +++ b/Gemfile @@ -16,7 +16,7 @@ gem 'puma' # assets. No es necesario instalarlas en producción. # # XXX: Supuestamente Rails ya soporta RAILS_GROUPS, pero Bundler no. -if ENV['RAILS_GROUPS']&.include? 'assets' +if ENV['RAILS_GROUPS']&.split(',')&.include? 'assets' gem 'sassc-rails' gem 'uglifier', '>= 1.3.0' gem 'bootstrap', '~> 4' From 25fe41bfa258d167c978152939bc65c9eb4c03b7 Mon Sep 17 00:00:00 2001 From: f Date: Wed, 27 Oct 2021 10:12:28 -0300 Subject: [PATCH 50/52] enviar el idioma al reordenar! MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit los tests envían el idioma pero no el panel --- app/views/posts/index.haml | 1 + 1 file changed, 1 insertion(+) diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index ad07b9dc..90d65670 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -67,6 +67,7 @@ %h2= t('posts.empty') - else = form_tag site_posts_reorder_path, method: :post do + %input{ type: 'hidden', name: 'post[lang]', value: @locale } %table.table{ data: { controller: 'reorder' } } %caption.sr-only= t('posts.caption') %thead From 21f650fc575656e174a22d36d3f89f6ea0e33f7f Mon Sep 17 00:00:00 2001 From: Nulo Date: Thu, 28 Oct 2021 16:13:39 -0300 Subject: [PATCH 51/52] Editor: forzar modo claro MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Esto hace que sea legible y más usable cuando el modo oscuro está activado. https://0xacab.org/sutty/sutty/-/issues/2135 Idealmente, me gustaría tener modo oscuro real en el editor. --- app/assets/stylesheets/editor.scss | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/assets/stylesheets/editor.scss b/app/assets/stylesheets/editor.scss index 5d218c7e..e0886533 100644 --- a/app/assets/stylesheets/editor.scss +++ b/app/assets/stylesheets/editor.scss @@ -2,6 +2,13 @@ box-sizing: border-box; *, *::before, *::after { box-sizing: inherit; } + // Arreglo temporal para que las cosas sean legibles en modo oscuro + --foreground: black; + --background: white; + --color: #f206f9; + background: var(--background); + color: var(--foreground); + h1, h2, h3, h4, h5, h6, p, li { min-height: 1.5rem; } From 6dcbe97ac6273fdf637f6143ecc1f88cb0255193 Mon Sep 17 00:00:00 2001 From: f Date: Thu, 4 Nov 2021 10:58:49 -0300 Subject: [PATCH 52/52] permalinks relativos --- app/models/metadata_permalink.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/metadata_permalink.rb b/app/models/metadata_permalink.rb index dc65aa86..59b68461 100644 --- a/app/models/metadata_permalink.rb +++ b/app/models/metadata_permalink.rb @@ -5,7 +5,7 @@ class MetadataPermalink < MetadataString # El valor por defecto una vez creado es la URL que le asigne Jekyll, # de forma que nunca cambia aunque se cambie el título. def default_value - document.url unless post.new? + document.url.sub(%r{\A/}, '') unless post.new? end # Los permalinks nunca pueden ser privados