Merge branch 'blazer' into panel.sutty.nl

app/controllers/stats_controller.rb

@@ -8,6 +8,10 @@ class StatsController < ApplicationController
   before_action :authenticate_usuarie!
   before_action :authorize_stats
 
+  breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path
+  breadcrumb 'sites.index', :sites_path, match: :exact
+  breadcrumb -> { site.title }, -> { site_posts_path(site, locale: locale) }, match: :exact
+
   EXTRA_OPTIONS = {
     builds: {},
     space_used: { bytes: true },
@@ -20,7 +24,16 @@ class StatsController < ApplicationController
     policy.script_src :self, :unsafe_inline
   end
 
+  # Parámetros por defecto
+  #
+  # @return [Hash]
+  def default_url_options
+    { interval: 'day', period_start: Date.today.beginning_of_year, period_end: Date.today }
+  end
+
   def index
+    breadcrumb I18n.t('stats.index.title'), ''
+
     @chart_params = { interval: interval }
     hostnames
     last_stat
@@ -30,9 +43,9 @@ class StatsController < ApplicationController
 
   # Genera un gráfico de visitas por dominio asociado a este sitio
   def host
-    return unless stale? [last_stat, hostnames, interval]
+    return unless stale? [last_stat, hostnames, interval, period]
 
-    stats = Rollup.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series|
+    stats = rollup_scope.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series|
       series.each do |serie|
         serie[:name] = serie.dig(:dimensions, 'host')
         serie[:data].transform_values! do |value|
@@ -45,7 +58,7 @@ class StatsController < ApplicationController
   end
 
   def resources
-    return unless stale? [last_stat, interval, resource]
+    return unless stale? [last_stat, interval, resource, period]
 
     options = {
       interval: interval,
@@ -54,14 +67,14 @@ class StatsController < ApplicationController
       }
     }
 
-    render json: Rollup.series(resource, **options)
+    render json: rollup_scope.series(resource, **options)
   end
 
   def uris
-    return unless stale? [last_stat, hostnames, interval, normalized_urls]
+    return unless stale? [last_stat, hostnames, interval, normalized_urls, period]
 
     options = { host: hostnames, uri: normalized_paths }
-    stats = Rollup.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series|
+    stats = rollup_scope.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series|
       series.each do |serie|
         serie[:name] = serie[:dimensions].slice('host', 'uri').values.join.sub('/index.html', '/')
         serie[:data].transform_values! do |value|
@@ -75,34 +88,43 @@ class StatsController < ApplicationController
 
   private
 
+  def rollup_scope
+    Rollup.where(time: period)
+  end
+
   def last_stat
-    @last_stat ||= Stat.last
+    @last_stat ||= site.stats.last
   end
 
   def authorize_stats
-    @site = find_site
+    authorize SiteStat.new(site)
-    authorize SiteStat.new(@site)
   end
 
   # TODO: Eliminar cuando mergeemos referer-origin
   def hostnames
-    @hostnames ||= [@site.hostname, @site.alternative_hostnames].flatten
+    @hostnames ||= site.hostnames
   end
 
   # Normalizar las URLs
   #
   # @return [Array]
   def normalized_urls
-    @normalized_urls ||= params.permit(:urls).try(:[],
+    @normalized_urls ||=
-                                                  :urls)&.split("\n")&.map(&:strip)&.select(&:present?)&.select do |uri|
+      begin
-                           uri.start_with? 'https://'
+        urls = params.permit(:urls).try(:[], :urls)&.split("\n")&.map(&:strip)&.select(&:present?)&.select do |uri|
-                         end&.map do |u|
+          uri.start_with? 'https://'
-                           # XXX: Eliminar
+        end
-                           # @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1}
-                           next u unless u.end_with? '/'
 
-                           "#{u}index.html"
+        urls ||= [site.url]
-                         end&.uniq || [@site.url, @site.urls].flatten.uniq
+
+        urls.map do |u|
+          # XXX: Eliminar al deployear
+          # @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1}
+          next u unless u.end_with? '/'
+
+          "#{u}index.html"
+        end.uniq
+      end
   end
 
   def normalized_paths
@@ -140,14 +162,15 @@ class StatsController < ApplicationController
   def interval
     @interval ||= begin
       i = params[:interval]&.to_sym
-      Stat::INTERVALS.include?(i) ? i : :day
+      Stat::INTERVALS.include?(i) ? i : Stat::INTERVALS.first
     end
   end
 
+  # @return [Symbol]
   def resource
     @resource ||= begin
       r = params[:resource].to_sym
-      Stat::RESOURCES.include?(r) ? r : :builds
+      Stat::RESOURCES.include?(r) ? r : Stat::RESOURCES.first
     end
   end
 
@@ -165,4 +188,15 @@ class StatsController < ApplicationController
   def nodes
     @nodes ||= ENV.fetch('NODES', 1).to_i
   end
+
+  def period
+    @period ||= begin
+      p = params.permit(:period_start, :period_end)
+      p[:period_start]..p[:period_end]
+    end
+  end
+
+  def site
+    @site ||= find_site
+  end
 end

app/jobs/concerns/recursive_rollup.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+# Implementa rollups recursivos
 module RecursiveRollup
   extend ActiveSupport::Concern
 
@@ -18,9 +19,8 @@ module RecursiveRollup
     # @param :beginning [Time]
     # @return [Rollup]
     def recursive_rollup(name:, interval_previous:, interval:, dimensions:, beginning:, operation: :sum, new_name: nil)
-      Rollup.where(name: name, interval: interval_previous)
+      Rollup.where(name: name, interval: interval_previous, dimensions: dimensions.to_json)
             .where('time >= ?', beginning.try(:"beginning_of_#{interval}"))
-            .where_dimensions(**dimensions)
             .group(*dimensions_to_jsonb_query(dimensions))
             .rollup(new_name || name, interval: interval, update: true) do |rollup|
               rollup.try(operation, :value)

app/jobs/uri_collection_job.rb

@@ -20,22 +20,30 @@ class UriCollectionJob < PeriodicJob
 
   def perform(site_id:, once: true)
     @site = Site.find site_id
+    return unless File.directory? destination
 
     # Recordar la última vez que se corrió la tarea
     stat = site.stats.create! name: STAT_NAME
-    name = 'host|uri'
     beginning = beginning_of_interval
 
-    hostnames.each do |host|
+    site.hostnames.each do |host|
       break if stop?
 
       uris.each do |uri|
         break if stop?
 
-        rollup_uri(uri, host, name, beginning)
+        rollup('host|uri', beginning, host: host, uri: uri)
+
+        AccessLog.where(host: host, uri: uri).distinct(:http_referer).pluck(:http_referer).each do |http_referer|
+          rollup('host|uri|referer', beginning, host: host, uri: uri, http_referer: http_referer)
+        end
       end
 
-      rollup_host(host, name, beginning)
+      rollup('host', beginning, host: host)
+
+      AccessLog.where(host: host).distinct(:http_referer).pluck(:http_referer).each do |http_referer|
+        rollup('host|referer', beginning, host: host, http_referer: http_referer)
+      end
     end
 
     stat.touch
@@ -45,9 +53,14 @@ class UriCollectionJob < PeriodicJob
 
   private
 
-  def rollup_uri(uri, host, name, beginning)
+  # Generar un rollup a partir de unas dimensiones que también sirven de
-    dimensions = { host: host, uri: uri }
+  # filtro.
-
+  #
+  # @param :name [String]
+  # @param :beginning [Time]
+  # @param :dimensions [Hash]
+  # @return [nil]
+  def rollup(name, beginning, **dimensions)
     AccessLog.where(**dimensions)
              .where('created_at >= ?', beginning)
              .completed_requests
@@ -69,28 +82,6 @@ class UriCollectionJob < PeriodicJob
     end
   end
 
-  def rollup_host(host, name, beginning)
-    dimensions = { host: host }
-    new_name = 'host'
-
-    recursive_rollup(name: name,
-                     new_name: new_name,
-                     interval_previous: starting_interval,
-                     interval: starting_interval,
-                     dimensions: dimensions,
-                     beginning: beginning)
-
-    Stat::INTERVALS.reduce do |previous, current|
-      recursive_rollup(name: new_name,
-                       interval_previous: previous,
-                       interval: current,
-                       dimensions: dimensions,
-                       beginning: beginning)
-
-      current
-    end
-  end
-
   def stat_name
     STAT_NAME
   end
@@ -102,26 +93,6 @@ class UriCollectionJob < PeriodicJob
     @destination ||= site.deploys.find_by(type: 'DeployLocal').destination
   end
 
-  # TODO: Cambiar al mergear origin-referer
-  #
-  # @return [Array]
-  def hostnames
-    @hostnames ||= site.deploys.map do |deploy|
-      case deploy
-      when DeployLocal
-        site.hostname
-      when DeployWww
-        deploy.fqdn
-      when DeployAlternativeDomain
-        deploy.hostname.dup.tap do |h|
-          h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}")
-        end
-      when DeployHiddenService
-        deploy.onion
-      end
-    end.compact
-  end
-
   # Recolecta todas las URIs menos imágenes
   #
   # @return [Array]

app/models/site.rb

@@ -101,6 +101,26 @@ class Site < ApplicationRecord
     "https://#{hostname}#{slash ? '/' : ''}"
   end
 
+  # TODO: Cambiar al mergear origin-referer
+  #
+  # @return [Array]
+  def hostnames
+    @hostnames ||= deploys.map do |deploy|
+      case deploy
+      when DeployLocal
+        hostname
+      when DeployWww
+        deploy.fqdn
+      when DeployAlternativeDomain
+        deploy.hostname.dup.tap do |h|
+          h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}")
+        end
+      when DeployHiddenService
+        deploy.onion
+      end
+    end.compact
+  end
+
   # Obtiene los dominios alternativos
   #
   # @return Array
@@ -123,7 +143,9 @@ class Site < ApplicationRecord
   #
   # @return Array
   def urls(slash: true)
-    alternative_urls(slash: slash) << url(slash: slash)
+    @urls ||= hostnames.map do |h|
+      "https://#{h}#{slash ? '/' : ''}"
+    end
   end
 
   def invitade?(usuarie)

app/views/stats/index.haml

@@ -25,12 +25,12 @@
         %input{ type: 'hidden', name: 'interval', value: @interval }
         .form-group
           %label{ for: 'urls' }= t('.urls.label')
-          %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size, aria_describedby: 'help-urls' }= @normalized_urls.join("\n")
+          %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size + 1, aria_describedby: 'help-urls' }= @normalized_urls.join("\n")
           %small#help-urls.feedback.form-text.text-muted= t('.urls.help')
         .form-group
           %button.btn{ type: 'submit' }= t('.urls.submit')
       - if @normalized_urls.present?
-        = line_chart site_stats_uris_path(urls: params[:urls], **@chart_params), **@chart_options
+        = line_chart site_stats_uris_path(urls: @normalized_urls, **@chart_params), **@chart_options
 
     .mb-5
       %h2= t('.resources.title')