From 298fd5bed14fa31c04dcf8d9cb5785c8c86c3c36 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:46:48 -0300 Subject: [PATCH 01/11] centralizar la forma en que se obtienen los hostnames relacionado con !50 --- app/jobs/uri_collection_job.rb | 22 +--------------------- app/models/site.rb | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index cad05cbb..ea968fa0 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -26,7 +26,7 @@ class UriCollectionJob < PeriodicJob name = 'host|uri' beginning = beginning_of_interval - hostnames.each do |host| + site.hostnames.each do |host| break if stop? uris.each do |uri| @@ -102,26 +102,6 @@ class UriCollectionJob < PeriodicJob @destination ||= site.deploys.find_by(type: 'DeployLocal').destination end - # TODO: Cambiar al mergear origin-referer - # - # @return [Array] - def hostnames - @hostnames ||= site.deploys.map do |deploy| - case deploy - when DeployLocal - site.hostname - when DeployWww - deploy.fqdn - when DeployAlternativeDomain - deploy.hostname.dup.tap do |h| - h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}") - end - when DeployHiddenService - deploy.onion - end - end.compact - end - # Recolecta todas las URIs menos imágenes # # @return [Array] diff --git a/app/models/site.rb b/app/models/site.rb index 5b78d625..bcc9e60b 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -101,6 +101,26 @@ class Site < ApplicationRecord "https://#{hostname}#{slash ? '/' : ''}" end + # TODO: Cambiar al mergear origin-referer + # + # @return [Array] + def hostnames + @hostnames ||= deploys.map do |deploy| + case deploy + when DeployLocal + hostname + when DeployWww + deploy.fqdn + when DeployAlternativeDomain + deploy.hostname.dup.tap do |h| + h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}") + end + when DeployHiddenService + deploy.onion + end + end.compact + end + # Obtiene los dominios alternativos # # @return Array @@ -123,7 +143,9 @@ class Site < ApplicationRecord # # @return Array def urls(slash: true) - alternative_urls(slash: slash) << url(slash: slash) + @urls ||= hostnames.map do |h| + "https://#{h}#{slash ? '/' : ''}" + end end def invitade?(usuarie) From fb5cc0aaa7c10749d78ce18b2246c7b20945d273 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:48:25 -0300 Subject: [PATCH 02/11] breadcrumbs --- app/controllers/stats_controller.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 44073c1f..ba9113ff 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -8,6 +8,10 @@ class StatsController < ApplicationController before_action :authenticate_usuarie! before_action :authorize_stats + breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path + breadcrumb 'sites.index', :sites_path, match: :exact + breadcrumb -> { site.title }, -> { site_posts_path(site, locale: locale) }, match: :exact + EXTRA_OPTIONS = { builds: {}, space_used: { bytes: true }, @@ -21,6 +25,8 @@ class StatsController < ApplicationController end def index + breadcrumb I18n.t('stats.index.title'), '' + @chart_params = { interval: interval } hostnames last_stat From f72d27c3083ee32b50da0baa8e948566de65e79f Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:49:26 -0300 Subject: [PATCH 03/11] =?UTF-8?q?optimizaci=C3=B3n:=20usar=20las=20dimensi?= =?UTF-8?q?ones=20como=20=C3=ADndices=20y=20no=20como=20texto?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Rollup#where_dimensions` no aprovecha los índices. --- app/jobs/concerns/recursive_rollup.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb index 3163558e..b3327a73 100644 --- a/app/jobs/concerns/recursive_rollup.rb +++ b/app/jobs/concerns/recursive_rollup.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# Implementa rollups recursivos module RecursiveRollup extend ActiveSupport::Concern @@ -18,9 +19,8 @@ module RecursiveRollup # @param :beginning [Time] # @return [Rollup] def recursive_rollup(name:, interval_previous:, interval:, dimensions:, beginning:, operation: :sum, new_name: nil) - Rollup.where(name: name, interval: interval_previous) + Rollup.where(name: name, interval: interval_previous, dimensions: dimensions.to_json) .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) - .where_dimensions(**dimensions) .group(*dimensions_to_jsonb_query(dimensions)) .rollup(new_name || name, interval: interval, update: true) do |rollup| rollup.try(operation, :value) From 7c7ae2f75041b667a8f77ecbbf43fe2c569809d8 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:52:27 -0300 Subject: [PATCH 04/11] =?UTF-8?q?mostrar=20un=20per=C3=ADodo=20por=20defec?= =?UTF-8?q?to?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit todavía no hay interfaz para cambiarlos --- app/controllers/stats_controller.rb | 30 +++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index ba9113ff..ded3a05c 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -24,6 +24,13 @@ class StatsController < ApplicationController policy.script_src :self, :unsafe_inline end + # Parámetros por defecto + # + # @return [Hash] + def default_url_options + { interval: 'day', period_start: Date.today.beginning_of_year, period_end: Date.today } + end + def index breadcrumb I18n.t('stats.index.title'), '' @@ -36,9 +43,9 @@ class StatsController < ApplicationController # Genera un gráfico de visitas por dominio asociado a este sitio def host - return unless stale? [last_stat, hostnames, interval] + return unless stale? [last_stat, hostnames, interval, period] - stats = Rollup.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series| + stats = rollup_scope.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series| series.each do |serie| serie[:name] = serie.dig(:dimensions, 'host') serie[:data].transform_values! do |value| @@ -51,7 +58,7 @@ class StatsController < ApplicationController end def resources - return unless stale? [last_stat, interval, resource] + return unless stale? [last_stat, interval, resource, period] options = { interval: interval, @@ -60,14 +67,14 @@ class StatsController < ApplicationController } } - render json: Rollup.series(resource, **options) + render json: rollup_scope.series(resource, **options) end def uris - return unless stale? [last_stat, hostnames, interval, normalized_urls] + return unless stale? [last_stat, hostnames, interval, normalized_urls, period] options = { host: hostnames, uri: normalized_paths } - stats = Rollup.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series| + stats = rollup_scope.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series| series.each do |serie| serie[:name] = serie[:dimensions].slice('host', 'uri').values.join.sub('/index.html', '/') serie[:data].transform_values! do |value| @@ -81,6 +88,10 @@ class StatsController < ApplicationController private + def rollup_scope + Rollup.where(time: period) + end + def last_stat @last_stat ||= Stat.last end @@ -171,4 +182,11 @@ class StatsController < ApplicationController def nodes @nodes ||= ENV.fetch('NODES', 1).to_i end + + def period + @period ||= begin + p = params.permit(:period_start, :period_end) + p[:period_start]..p[:period_end] + end + end end From 3a32422e26695feb58d4c45b5526d1bca227dfb8 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:54:04 -0300 Subject: [PATCH 05/11] =?UTF-8?q?usar=20m=C3=A9todos=20del=20sitio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index ded3a05c..116bca7c 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -103,23 +103,29 @@ class StatsController < ApplicationController # TODO: Eliminar cuando mergeemos referer-origin def hostnames - @hostnames ||= [@site.hostname, @site.alternative_hostnames].flatten + @hostnames ||= site.hostnames end # Normalizar las URLs # # @return [Array] def normalized_urls - @normalized_urls ||= params.permit(:urls).try(:[], - :urls)&.split("\n")&.map(&:strip)&.select(&:present?)&.select do |uri| - uri.start_with? 'https://' - end&.map do |u| - # XXX: Eliminar - # @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1} - next u unless u.end_with? '/' + @normalized_urls ||= + begin + urls = params.permit(:urls).try(:[], :urls)&.split("\n")&.map(&:strip)&.select(&:present?)&.select do |uri| + uri.start_with? 'https://' + end - "#{u}index.html" - end&.uniq || [@site.url, @site.urls].flatten.uniq + urls ||= [site.url] + + urls.map do |u| + # XXX: Eliminar al deployear + # @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1} + next u unless u.end_with? '/' + + "#{u}index.html" + end.uniq + end end def normalized_paths From 027d24d66a83dbd00273c3310e121090ade72097 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:55:54 -0300 Subject: [PATCH 06/11] memoizar el sitio --- app/controllers/stats_controller.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 116bca7c..10b75819 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -97,8 +97,7 @@ class StatsController < ApplicationController end def authorize_stats - @site = find_site - authorize SiteStat.new(@site) + authorize SiteStat.new(site) end # TODO: Eliminar cuando mergeemos referer-origin @@ -195,4 +194,8 @@ class StatsController < ApplicationController p[:period_start]..p[:period_end] end end + + def site + @site ||= find_site + end end From 0754e29a5a272a276ebefb93ab620838e9cc2fca Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:56:25 -0300 Subject: [PATCH 07/11] =?UTF-8?q?cada=20sitio=20tiene=20su=20propio=20cont?= =?UTF-8?q?rol=20de=20estad=C3=ADsticas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/controllers/stats_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 10b75819..4933c821 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -93,7 +93,7 @@ class StatsController < ApplicationController end def last_stat - @last_stat ||= Stat.last + @last_stat ||= site.stats.last end def authorize_stats From 7e575372cf1a7a42e4817f2ebe858941163daf6a Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:56:45 -0300 Subject: [PATCH 08/11] parametros por defecto --- app/controllers/stats_controller.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 4933c821..9a8a345d 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -162,14 +162,15 @@ class StatsController < ApplicationController def interval @interval ||= begin i = params[:interval]&.to_sym - Stat::INTERVALS.include?(i) ? i : :day + Stat::INTERVALS.include?(i) ? i : Stat::INTERVALS.first end end + # @return [Symbol] def resource @resource ||= begin r = params[:resource].to_sym - Stat::RESOURCES.include?(r) ? r : :builds + Stat::RESOURCES.include?(r) ? r : Stat::RESOURCES.first end end From a703eb409643fa6fa0d068d596472e7cfff5dac3 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:57:16 -0300 Subject: [PATCH 09/11] no fallar si el sitio no existe --- app/jobs/uri_collection_job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index ea968fa0..796a289c 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -20,6 +20,7 @@ class UriCollectionJob < PeriodicJob def perform(site_id:, once: true) @site = Site.find site_id + return unless File.directory? destination # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME From 5186892a87c0ec15aa1a4f9cb1dafa1b5173444c Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:57:57 -0300 Subject: [PATCH 10/11] =?UTF-8?q?fixup!=20usar=20m=C3=A9todos=20del=20siti?= =?UTF-8?q?o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/views/stats/index.haml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index bfcf33ef..2b5d643e 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -25,12 +25,12 @@ %input{ type: 'hidden', name: 'interval', value: @interval } .form-group %label{ for: 'urls' }= t('.urls.label') - %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size, aria_describedby: 'help-urls' }= @normalized_urls.join("\n") + %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size + 1, aria_describedby: 'help-urls' }= @normalized_urls.join("\n") %small#help-urls.feedback.form-text.text-muted= t('.urls.help') .form-group %button.btn{ type: 'submit' }= t('.urls.submit') - if @normalized_urls.present? - = line_chart site_stats_uris_path(urls: params[:urls], **@chart_params), **@chart_options + = line_chart site_stats_uris_path(urls: @normalized_urls, **@chart_params), **@chart_options .mb-5 %h2= t('.resources.title') From be6224ddf173f35d45a26df68abe71aea44282f2 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 26 Apr 2022 16:59:02 -0300 Subject: [PATCH 11/11] =?UTF-8?q?recolectar=20estad=C3=ADsticas=20de=20ref?= =?UTF-8?q?erers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/jobs/uri_collection_job.rb | 46 +++++++++++++--------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 796a289c..12f25eb6 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -24,7 +24,6 @@ class UriCollectionJob < PeriodicJob # Recordar la última vez que se corrió la tarea stat = site.stats.create! name: STAT_NAME - name = 'host|uri' beginning = beginning_of_interval site.hostnames.each do |host| @@ -33,10 +32,18 @@ class UriCollectionJob < PeriodicJob uris.each do |uri| break if stop? - rollup_uri(uri, host, name, beginning) + rollup('host|uri', beginning, host: host, uri: uri) + + AccessLog.where(host: host, uri: uri).distinct(:http_referer).pluck(:http_referer).each do |http_referer| + rollup('host|uri|referer', beginning, host: host, uri: uri, http_referer: http_referer) + end end - rollup_host(host, name, beginning) + rollup('host', beginning, host: host) + + AccessLog.where(host: host).distinct(:http_referer).pluck(:http_referer).each do |http_referer| + rollup('host|referer', beginning, host: host, http_referer: http_referer) + end end stat.touch @@ -46,9 +53,14 @@ class UriCollectionJob < PeriodicJob private - def rollup_uri(uri, host, name, beginning) - dimensions = { host: host, uri: uri } - + # Generar un rollup a partir de unas dimensiones que también sirven de + # filtro. + # + # @param :name [String] + # @param :beginning [Time] + # @param :dimensions [Hash] + # @return [nil] + def rollup(name, beginning, **dimensions) AccessLog.where(**dimensions) .where('created_at >= ?', beginning) .completed_requests @@ -70,28 +82,6 @@ class UriCollectionJob < PeriodicJob end end - def rollup_host(host, name, beginning) - dimensions = { host: host } - new_name = 'host' - - recursive_rollup(name: name, - new_name: new_name, - interval_previous: starting_interval, - interval: starting_interval, - dimensions: dimensions, - beginning: beginning) - - Stat::INTERVALS.reduce do |previous, current| - recursive_rollup(name: new_name, - interval_previous: previous, - interval: current, - dimensions: dimensions, - beginning: beginning) - - current - end - end - def stat_name STAT_NAME end