Merge branch 'blazer' into 'rails'

Mostrar estadísticas en el panel de Sutty

Closes #2331, #7530, #7529, #7528, #7527, #7512, #7511, #7510, #7509, #7494, #7493, #7492, #7491, #7408, #7407, #7406, #7405, #7397, #7396, #7395, #7394, #7383, #7382, #7381, #7380, #7170, #7169, #7168, #7167, #7154, #7153, #7152, and #7151

See merge request sutty/sutty!52

Gemfile

@@ -64,7 +64,7 @@ gem 'rails-i18n'
 gem 'rails_warden'
 gem 'redis', require: %w[redis redis/connection/hiredis]
 gem 'redis-rails'
-gem 'rollups', git: 'https://github.com/ankane/rollup.git', branch: 'master'
+gem 'rollups', git: 'https://github.com/fauno/rollup.git', branch: 'update'
 gem 'rubyzip'
 gem 'rugged'
 gem 'concurrent-ruby-ext'

Gemfile.lock

@@ -6,15 +6,6 @@ GIT
       rails (>= 3.0)
       rake (>= 0.8.7)
 
-GIT
-  remote: https://github.com/ankane/rollup.git
-  revision: 0ab6c603450175eb1004f7793e86486943cb9f72
-  branch: master
-  specs:
-    rollups (0.1.3)
-      activesupport (>= 5.1)
-      groupdate (>= 5.2)
-
 GIT
   remote: https://github.com/fauno/email_address
   revision: 536b51f7071b68a55140c0c1726b4cd401d1c04d
@@ -24,6 +15,15 @@ GIT
       netaddr (>= 2.0.4, < 3)
       simpleidn
 
+GIT
+  remote: https://github.com/fauno/rollup.git
+  revision: ddbb345aa57e63b4cfdf7557267efa89ba60caac
+  branch: update
+  specs:
+    rollups (0.1.3)
+      activesupport (>= 5.1)
+      groupdate (>= 5.2)
+
 GEM
   remote: https://gems.sutty.nl/
   specs:
@@ -214,8 +214,8 @@ GEM
       ffi (~> 1.0)
     globalid (0.6.0)
       activesupport (>= 5.0)
-    groupdate (5.2.2)
-      activesupport (>= 5)
+    groupdate (6.1.0)
+      activesupport (>= 5.2)
     hairtrigger (0.2.24)
       activerecord (>= 5.0, < 7)
       ruby2ruby (~> 2.4)

Procfile

@@ -5,3 +5,4 @@ blazer_1h: bundle exec rake blazer:run_checks SCHEDULE="1 hour"
 blazer_1d: bundle exec rake blazer:run_checks SCHEDULE="1 day"
 blazer: bundle exec rake blazer:send_failing_checks
 prometheus: bundle exec prometheus_exporter -b 0.0.0.0 --prefix "sutty_"
+stats: bundle exec rake stats:process_all

app/assets/stylesheets/application.scss

@@ -126,6 +126,7 @@ ol.breadcrumb {
   color: var(--foreground);
 }
 
+.table tr.sticky-top,
 .form-control,
 .custom-file-label {
   background-color: var(--background);

app/controllers/application_controller.rb

@@ -3,6 +3,7 @@
 # Forma de ingreso a Sutty
 class ApplicationController < ActionController::Base
   include ExceptionHandler
+  include Pundit
 
   protect_from_forgery with: :null_session, prepend: true
 
@@ -10,6 +11,7 @@ class ApplicationController < ActionController::Base
   before_action :configure_permitted_parameters, if: :devise_controller?
   around_action :set_locale
 
+  rescue_from Pundit::NilPolicyError, with: :page_not_found
   rescue_from ActionController::RoutingError, with: :page_not_found
   rescue_from ActionController::ParameterMissing, with: :page_not_found
 
@@ -33,7 +35,7 @@ class ApplicationController < ActionController::Base
   def find_site
     id = params[:site_id] || params[:id]
 
-    unless (site = current_usuarie.sites.find_by_name(id))
+    unless (site = current_usuarie&.sites&.find_by_name(id))
       raise SiteNotFound
     end
 
@@ -62,6 +64,21 @@ class ApplicationController < ActionController::Base
     render 'application/page_not_found', status: :not_found
   end
 
+  # Necesario para poder acceder a Blazer.  Solo les usuaries de este
+  # sitio pueden acceder al panel.
+  def require_usuarie
+    site = find_site
+    authorize SiteBlazer.new(site)
+
+    # Necesario para los breadcrumbs.
+    ActionView::Base.include Loaf::ViewExtensions unless ActionView::Base.included_modules.include? Loaf::ViewExtensions
+
+    breadcrumb current_usuarie.email, main_app.edit_usuarie_registration_path
+    breadcrumb 'sites.index', main_app.sites_path, match: :exact
+    breadcrumb site.title, main_app.site_path(site), match: :exact
+    breadcrumb 'stats.index', root_path, match: :exact
+  end
+
   protected
 
   def configure_permitted_parameters

app/controllers/concerns/blazer_decorator.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+# Modificaciones para Blazer
+module BlazerDecorator
+  # No poder obtener información de la base de datos.
+  module DisableDatabaseInfo
+    extend ActiveSupport::Concern
+
+    included do
+      def docs; end
+
+      def tables; end
+
+      def schema; end
+    end
+  end
+
+  # Deshabilitar edición de consultas y chequeos.
+  module DisableEdits
+    extend ActiveSupport::Concern
+
+    included do
+      def create; end
+
+      def update; end
+
+      def destroy; end
+
+      def run; end
+
+      def refresh; end
+
+      def cancel; end
+    end
+  end
+
+  # Blazer hace un gran esfuerzo para ejecutar consultas de forma
+  # asincrónica pero termina enviándolas por JS.
+  module RunSync
+    extend ActiveSupport::Concern
+
+    included do
+      alias_method :original_show, :show
+
+      include Blazer::BaseHelper
+
+      def show
+        original_show
+
+        options = { user: blazer_user, query: @query, run_id: SecureRandom.uuid, async: false }
+        @data_source = Blazer.data_sources[@query.data_source]
+        @result = Blazer::RunStatement.new.perform(@data_source, @statement, options)
+        chart_data
+      end
+
+      private
+
+      # Solo mostrar las consultas de le usuarie
+      def set_queries(_ = nil)
+        @queries = (@current_usuarie || current_usuarie).blazer_queries
+      end
+
+      # blazer-2.4.2/app/views/blazer/queries/run.html.erb
+      def chart_type
+        case @result.chart_type
+        when /\Aline(2)?\z/
+          chart_options.merge! min: nil
+        when /\Abar(2)?\z/
+          chart_options.merge! library: { tooltips: { intersect: false, axis: 'x' } }
+        when 'pie'
+          chart_options
+        when 'scatter'
+          chart_options.merge! library: { tooltips: { intersect: false } }, xtitle: @result.columns[0],
+                               ytitle: @result.columns[1]
+        when nil
+        else
+          if @result.column_types.size == 2
+            chart_options.merge! library: { tooltips: { intersect: false, axis: 'x' } }
+          else
+            chart_options.merge! library: { tooltips: { intersect: false } }
+          end
+        end
+
+        @result.chart_type
+      end
+
+      def chart_data
+        @chart_data ||=
+          case chart_type
+          when 'line'
+            @result.columns[1..-1].each_with_index.map do |k, i|
+              {
+                name: blazer_series_name(k),
+                data: @result.rows.map do |r|
+                  [r[0], r[i + 1]]
+                end,
+                library: series_library[i]
+              }
+            end
+          when 'line2'
+            @result.rows.group_by do |r|
+              v = r[1]
+              (@result.boom[@result.columns[1]] || {})[v.to_s] || v
+            end.each_with_index.map do |(name, v), i|
+              {
+                name: blazer_series_name(name),
+                data: v.map do |v2|
+                  [v2[0], v2[2]]
+                end,
+                library: series_library[i]
+              }
+            end
+          when 'pie'
+            @result.rows.map do |r|
+              [(@result.boom[@result.columns[0]] || {})[r[0].to_s] || r[0], r[1]]
+            end
+          when 'bar'
+            (@result.rows.first.size - 1).times.map do |i|
+              name = @result.columns[i + 1]
+
+              {
+                name: blazer_series_name(name),
+                data: @result.rows.first(20).map do |r|
+                  [(@result.boom[@result.columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]]
+                end
+              }
+            end
+          when 'bar2'
+            first_20 = @result.rows.group_by { |r| r[0] }.values.first(20).flatten(1)
+            labels = first_20.map { |r| r[0] }.uniq
+            series = first_20.map { |r| r[1] }.uniq
+            labels.each do |l|
+              series.each do |s|
+                first_20 << [l, s, 0] unless first_20.find { |r| r[0] == l && r[1] == s }
+              end
+            end
+
+            first_20.group_by do |r|
+              v = r[1]
+              (@result.boom[@result.columns[1]] || {})[v.to_s] || v
+            end.each_with_index.map do |(name, v), _i|
+              {
+                name: blazer_series_name(name),
+                data: v.sort_by do |r2|
+                  labels.index(r2[0])
+                end.map do |v2|
+                  v3 = v2[0]
+                  [(@result.boom[@result.columns[0]] || {})[v3.to_s] || v3, v2[2]]
+                end
+              }
+            end
+          when 'scatter'
+            @result.rows
+          end
+      end
+
+      def target_index
+        @target_index ||= @result.columns.index do |k|
+          k.downcase == 'target'
+        end
+      end
+
+      def series_library
+        @series_library ||= {}.tap do |sl|
+          if target_index
+            color = '#109618'
+            sl[target_index - 1] = {
+              pointStyle: 'line',
+              hitRadius: 5,
+              borderColor: color,
+              pointBackgroundColor: color,
+              backgroundColor: color,
+              pointHoverBackgroundColor: color
+            }
+          end
+        end
+      end
+
+      def chart_options
+        @chart_options ||= { id: SecureRandom.hex }
+      end
+    end
+  end
+end
+
+classes = [Blazer::QueriesController, Blazer::ChecksController, Blazer::DashboardsController]
+modules = [BlazerDecorator::DisableDatabaseInfo, BlazerDecorator::DisableEdits]
+classes.each do |klass|
+  modules.each do |modul|
+    klass.include modul unless klass.included_modules.include? modul
+  end
+end
+
+Blazer::QueriesController.include BlazerDecorator::RunSync

app/controllers/posts_controller.rb

@@ -2,9 +2,6 @@
 
 # Controlador para artículos
 class PostsController < ApplicationController
-  include Pundit
-  rescue_from Pundit::NilPolicyError, with: :page_not_found
-
   before_action :authenticate_usuarie!
   before_action :service_for_direct_upload, only: %i[new edit]
 
@@ -38,6 +35,8 @@ class PostsController < ApplicationController
 
     # Filtrar los posts que les invitades no pueden ver
     @usuarie = site.usuarie? current_usuarie
+
+    @site_stat = SiteStat.new(site)
   end
 
   def show

app/controllers/private_controller.rb

@@ -6,8 +6,6 @@ class PrivateController < ApplicationController
   # XXX: Permite ejecutar JS
   skip_forgery_protection
 
-  include Pundit
-
   # Enviar el archivo si existe, agregar una / al final siempre para no
   # romper las direcciones relativas.
   def show

app/controllers/sites_controller.rb

@@ -2,9 +2,6 @@
 
 # Controlador de sitios
 class SitesController < ApplicationController
-  include Pundit
-  rescue_from Pundit::NilPolicyError, with: :page_not_found
-
   before_action :authenticate_usuarie!
 
   breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path

app/controllers/stats_controller.rb

@@ -8,6 +8,10 @@ class StatsController < ApplicationController
   before_action :authenticate_usuarie!
   before_action :authorize_stats
 
+  breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path
+  breadcrumb 'sites.index', :sites_path, match: :exact
+  breadcrumb -> { site.title }, -> { site_posts_path(site, locale: locale) }, match: :exact
+
   EXTRA_OPTIONS = {
     builds: {},
     space_used: { bytes: true },
@@ -20,19 +24,53 @@ class StatsController < ApplicationController
     policy.script_src :self, :unsafe_inline
   end
 
+  # Parámetros por defecto
+  #
+  # @return [Hash]
+  def default_url_options
+    { interval: 'day', period_start: Date.today.beginning_of_year, period_end: Date.today }
+  end
+
   def index
-    @chart_params = { interval: interval }
+    breadcrumb I18n.t('stats.index.title'), ''
+
+    params.with_defaults! default_url_options
+
+    @chart_params = {
+      interval: interval,
+      period_start: params[:period_start],
+      period_end: params[:period_end]
+    }
+
     hostnames
     last_stat
     chart_options
     normalized_urls
+
+    expires_in = Time.now.try(:"end_of_#{Stat.default_interval}") - Time.now
+    @columns = {}
+
+    Stat::COLUMNS.each do |column|
+      @columns[column] =
+        Rails.cache.fetch("stats/#{column}/#{site.id}", expires_in: expires_in) do
+          rollup_scope.where(interval: interval, name: "host|#{column}")
+                      .where_dimensions(host: hostnames)
+                      .group("dimensions->>'#{column}'")
+                      .order('sum(value) desc')
+                      .sum(:value)
+                      .transform_values(&:to_i)
+                      .transform_values do |v|
+                        v * nodes
+                      end
+        end
+    end
   end
 
   # Genera un gráfico de visitas por dominio asociado a este sitio
   def host
-    return unless stale? [last_stat, hostnames, interval]
+    return unless stale? [last_stat, hostnames, interval, period]
 
-    stats = Rollup.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series|
+    stats = rollup_scope.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series|
       series.each do |serie|
         serie[:name] = serie.dig(:dimensions, 'host')
         serie[:data].transform_values! do |value|
@@ -45,23 +83,20 @@ class StatsController < ApplicationController
   end
 
   def resources
-    return unless stale? [last_stat, interval, resource]
+    return unless stale? [last_stat, interval, resource, period]
 
-    options = {
-      interval: interval,
-      dimensions: {
-        deploy_id: @site.deploys.where(type: 'DeployLocal').pluck(:id).first
-      }
-    }
+    options = { interval: interval, dimensions: { site_id: site.id } }
 
-    render json: Rollup.series(resource, **options)
+    render json: rollup_scope.series(resource, **options)
   end
 
   def uris
-    return unless stale? [last_stat, hostnames, interval, normalized_urls]
+    return unless stale? [last_stat, hostnames, interval, normalized_urls, period]
 
     options = { host: hostnames, uri: normalized_paths }
-    stats = Rollup.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series|
+    # XXX: where_dimensions es más corto pero no aprovecha los índices
+    # de Rollup
+    stats = rollup_scope.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series|
       series.each do |serie|
         serie[:name] = serie[:dimensions].slice('host', 'uri').values.join.sub('/index.html', '/')
         serie[:data].transform_values! do |value|
@@ -75,34 +110,44 @@ class StatsController < ApplicationController
 
   private
 
+  def rollup_scope
+    Rollup.where(time: period)
+  end
+
   def last_stat
-    @last_stat ||= Stat.last
+    @last_stat ||= site.stats.last
   end
 
   def authorize_stats
-    @site = find_site
-    authorize SiteStat.new(@site)
+    authorize SiteStat.new(site)
   end
 
   # TODO: Eliminar cuando mergeemos referer-origin
   def hostnames
-    @hostnames ||= [@site.hostname, @site.alternative_hostnames].flatten
+    @hostnames ||= site.hostnames
   end
 
   # Normalizar las URLs
   #
   # @return [Array]
   def normalized_urls
-    @normalized_urls ||= params.permit(:urls).try(:[],
-                                                  :urls)&.split("\n")&.map(&:strip)&.select(&:present?)&.select do |uri|
-                           uri.start_with? 'https://'
-                         end&.map do |u|
-                           # XXX: Eliminar
-                           # @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1}
-                           next u unless u.end_with? '/'
+    @normalized_urls ||=
+      begin
+        urls = params[:urls].is_a?(Array) ? params[:urls] : params[:urls]&.split("\n")
+        urls = urls&.map(&:strip)&.select(&:present?)&.select do |uri|
+          uri.start_with? 'https://'
+        end
 
-                           "#{u}index.html"
-                         end&.uniq || [@site.url, @site.urls].flatten.uniq
+        urls ||= [site.url]
+
+        urls.map do |u|
+          # XXX: Eliminar al deployear
+          # @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1}
+          next u unless u.end_with? '/'
+
+          "#{u}index.html"
+        end.uniq
+      end
   end
 
   def normalized_paths
@@ -140,14 +185,15 @@ class StatsController < ApplicationController
   def interval
     @interval ||= begin
       i = params[:interval]&.to_sym
-      Stat::INTERVALS.include?(i) ? i : :day
+      Stat::INTERVALS.include?(i) ? i : Stat::INTERVALS.first
     end
   end
 
+  # @return [Symbol]
   def resource
     @resource ||= begin
       r = params[:resource].to_sym
-      Stat::RESOURCES.include?(r) ? r : :builds
+      Stat::RESOURCES.include?(r) ? r : Stat::RESOURCES.first
     end
   end
 
@@ -165,4 +211,15 @@ class StatsController < ApplicationController
   def nodes
     @nodes ||= ENV.fetch('NODES', 1).to_i
   end
+
+  def period
+    @period ||= begin
+      p = params.permit(:period_start, :period_end)
+      p[:period_start]..p[:period_end]
+    end
+  end
+
+  def site
+    @site ||= find_site
+  end
 end

app/jobs/concerns/recursive_rollup.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# Implementa rollups recursivos
+module RecursiveRollup
+  extend ActiveSupport::Concern
+
+  included do
+    private
+
+    # Genera un rollup recursivo en base al período anterior y aplica una
+    # operación.
+    #
+    # @param :name [String]
+    # @param :interval_previous [String]
+    # @param :interval [String]
+    # @param :operation [Symbol]
+    # @param :dimensions [Hash]
+    # @param :beginning [Time]
+    # @return [Rollup]
+    def recursive_rollup(name:, interval_previous:, interval:, dimensions:, beginning:, operation: :sum)
+      Rollup.where(name: name, interval: interval_previous, dimensions: dimensions)
+            .where('time >= ?', beginning.try(:"beginning_of_#{interval}"))
+            .group(*dimensions_to_jsonb_query(dimensions))
+            .rollup(name, interval: interval, update: true) do |rollup|
+              rollup.try(operation, :value)
+            end
+    end
+
+    # Reducir las estadísticas calculadas aplicando un rollup sobre el
+    # intervalo más amplio.
+    #
+    # @param :name [String]
+    # @param :operation [Symbol]
+    # @param :dimensions [Hash]
+    # @return [nil]
+    def reduce_rollup(name:, dimensions:, operation: :sum)
+      Stat::INTERVALS.reduce do |previous, current|
+        recursive_rollup(name: name,
+                         interval_previous: previous,
+                         interval: current,
+                         dimensions: dimensions,
+                         beginning: beginning_of_interval,
+                         operation: operation)
+
+        # Devolver el intervalo actual
+        current
+      end
+
+      nil
+    end
+
+    # @param :dimensions [Hash]
+    # @return [Array]
+    def dimensions_to_jsonb_query(dimensions)
+      dimensions.keys.map do |key|
+        "dimensions->'#{key}'"
+      end
+    end
+  end
+end

app/jobs/periodic_job.rb

@@ -52,4 +52,12 @@ class PeriodicJob < ApplicationJob
   def beginning_of_interval
     @beginning_of_interval ||= last_stat.created_at.try(:"beginning_of_#{starting_interval}")
   end
+
+  def stop_file
+    @stop_file ||= Rails.root.join('tmp', self.class.to_s.tableize)
+  end
+
+  def stop?
+    File.exist? stop_file
+  end
 end

app/jobs/stat_collection_job.rb

@@ -2,11 +2,15 @@
 
 # Genera resúmenes de información para poder mostrar estadísticas y se
 # corre regularmente a sí misma.
-class StatCollectionJob < ApplicationJob
+class StatCollectionJob < PeriodicJob
+  include RecursiveRollup
+
   STAT_NAME = 'stat_collection_job'
 
   def perform(site_id:, once: true)
     @site = Site.find site_id
+    beginning = beginning_of_interval
+    stat = site.stats.create! name: STAT_NAME
 
     scope.rollup('builds', **options)
 
@@ -18,44 +22,23 @@ class StatCollectionJob < ApplicationJob
       rollup.average(:seconds)
     end
 
-    # XXX: Es correcto promediar promedios?
-    Stat::INTERVALS.reduce do |previous, current|
-      rollup(name: 'builds', interval_previous: previous, interval: current)
-      rollup(name: 'space_used', interval_previous: previous, interval: current, operation: :average)
-      rollup(name: 'build_time', interval_previous: previous, interval: current, operation: :average)
+    dimensions = { site_id: site_id }
 
-      current
-    end
-
-    # Registrar que se hicieron todas las recolecciones
-    site.stats.create! name: STAT_NAME
+    reduce_rollup(name: 'builds', operation: :sum, dimensions: dimensions)
+    reduce_rollup(name: 'space_used', operation: :average, dimensions: dimensions)
+    reduce_rollup(name: 'build_time', operation: :average, dimensions: dimensions)
 
+    stat.touch
     run_again! unless once
   end
 
   private
 
-  # Genera un rollup recursivo en base al período anterior y aplica una
-  # operación.
-  #
-  # @return [NilClass]
-  def rollup(name:, interval_previous:, interval:, operation: :sum)
-    Rollup.where(name: name, interval: interval_previous)
-          .where_dimensions(site_id: site.id)
-          .group("dimensions->'site_id'")
-          .rollup(name, interval: interval, update: true) do |rollup|
-            rollup.try(:operation, :value)
-          end
-  end
-
   # Los registros a procesar
   #
   # @return [ActiveRecord::Relation]
   def scope
-    @scope ||= site.build_stats
-                   .jekyll
-                   .where('created_at => ?', beginning_of_interval)
-                   .group(:site_id)
+    @scope ||= site.build_stats.jekyll.where('build_stats.created_at >= ?', beginning_of_interval).group(:site_id)
   end
 
   # Las opciones por defecto
@@ -64,4 +47,8 @@ class StatCollectionJob < ApplicationJob
   def options
     @options ||= { interval: starting_interval, update: true }
   end
+
+  def stat_name
+    STAT_NAME
+  end
 end

app/jobs/uri_collection_job.rb

@@ -13,94 +13,160 @@
 class UriCollectionJob < PeriodicJob
   # Ignoramos imágenes porque suelen ser demasiadas y no aportan a las
   # estadísticas.
-  IMAGES = %w[.png .jpg .jpeg .gif .webp].freeze
+  IMAGES = %w[.png .jpg .jpeg .gif .webp .jfif].freeze
   STAT_NAME = 'uri_collection_job'
 
   def perform(site_id:, once: true)
     @site = Site.find site_id
 
-    hostnames.each do |hostname|
-      uris.each do |uri|
-        return if File.exist? Rails.root.join('tmp', 'uri_collection_job_stop')
+    # Obtener el principio del intervalo anterior
+    beginning_of_interval
+    # Recordar la última vez que se corrió la tarea
+    stat = site.stats.create! name: STAT_NAME
+    # Columnas a agrupar
+    columns = Stat::COLUMNS.zip([nil]).to_h
 
-        AccessLog.where(host: hostname, uri: uri)
-                 .where('created_at >= ?', beginning_of_interval)
-                 .completed_requests
-                 .non_robots
-                 .group(:host, :uri)
-                 .rollup('host|uri', interval: starting_interval, update: true)
+    # Las URIs son la fuente de verdad de las visitas, porque son las
+    # que indican las páginas y recursos descargables, el resto son
+    # imágenes, CSS, JS y tipografías que no nos aportan números
+    # significativos.
+    uri_dimensions = { host: site.hostnames, uri: uris }
+    host_dimensions = { host: site.hostnames }
 
-        # Reducir las estadísticas calculadas aplicando un rollup sobre el
-        # intervalo más amplio.
-        Stat::INTERVALS.reduce do |previous, current|
-          Rollup.where(name: 'host|uri', interval: previous)
-                .where_dimensions(host: hostname, uri: uri)
-                .group("dimensions->'host'", "dimensions->'uri'")
-                .rollup('host|uri', interval: current, update: true) do |rollup|
-                  rollup.sum(:value)
-                end
-
-          # Devolver el intervalo actual
-          current
-        end
-      end
+    # Recorremos todos los hostnames y uris posibles y luego agrupamos
+    # recursivamente para no tener que recalcular, asumiendo que es más
+    # rápido buscar en los rollups indexados que en la tabla en bruto.
+    #
+    # Los referers solo se agrupan por host.
+    columns.each_key do |column|
+      columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column)
     end
 
-    # Recordar la última vez que se corrió la tarea
-    site.stats.create! name: STAT_NAME
+    # Cantidad de visitas por host
+    rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions)
+    reduce_rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions)
+
+    # Cantidad de visitas por página/recurso
+    rollup(name: 'host|uri', dimensions: uri_dimensions)
+    reduce_rollup(name: 'host|uri', dimensions: uri_dimensions)
+
+    # Cantidad de visitas host y parámetro
+    columns.each_pair do |column, values|
+      column_name = "host|#{column}"
+      column_dimensions = { host: site.hostnames }
+      column_dimensions[column] = values
+
+      rollup(name: column_name, dimensions: column_dimensions, filter: uri_dimensions)
+      reduce_rollup(name: column_name, dimensions: column_dimensions)
+    end
+
+    stat.touch
 
     run_again! unless once
   end
 
   private
 
+  # Generar un rollup de access logs
+  #
+  # @param :name [String]
+  # @param :beginning [Time]
+  # @param :dimensions [Hash]
+  # @param :filter [Hash]
+  # @return [nil]
+  def rollup(name:, dimensions:, interval: starting_interval, filter: nil)
+    AccessLog.where(**(filter || dimensions))
+             .where('created_at >= ?', beginning_of_interval)
+             .completed_requests
+             .non_robots
+             .group(*dimensions.keys)
+             .rollup(name, interval: interval, update: true)
+  end
+
+  # Generar rollups con el resto de la información
+  #
+  # @param :name [String]
+  # @param :dimensions [Hash]
+  # @param :filter [Hash]
+  # @return [nil]
+  def reduce_rollup(name:, dimensions:, filter: nil)
+    Stat::INTERVALS.reduce do |_previous, current|
+      rollup(name: name, dimensions: dimensions, filter: filter, interval: current)
+
+      current
+    end
+
+    nil
+  end
+
   def stat_name
     STAT_NAME
   end
 
+  # Obtiene todas las ubicaciones de archivos
+  #
   # @return [String]
   #
   # TODO: Cambiar al mergear origin-referer
-  def destination
-    @destination ||= site.deploys.find_by(type: 'DeployLocal').destination
-  end
-
-  # TODO: Cambiar al mergear origin-referer
-  #
-  # @return [Array]
-  def hostnames
-    @hostnames ||= site.deploys.map do |deploy|
-      case deploy
-      when DeployLocal
-        site.hostname
-      when DeployWww
-        deploy.fqdn
-      when DeployAlternativeDomain
-        deploy.hostname.dup.tap do |h|
-          h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}")
-        end
-      when DeployHiddenService
-        deploy.onion
-      end
-    end.compact
+  def destinations
+    @destinations ||= site.deploys.map(&:destination).compact.select do |d|
+      File.directory?(d)
+    end.map do |d|
+      File.realpath(d)
+    end.uniq
   end
 
   # Recolecta todas las URIs menos imágenes
   #
+  # TODO: Para los sitios con DeployLocalizedDomain estamos buscando
+  # URIs de más.
+  #
   # @return [Array]
   def uris
-    @uris ||= Dir.chdir destination do
-      (Dir.glob('**/*.html') + Dir.glob('public/**/*').reject do |p|
-        File.directory? p
-      end.reject do |p|
-        p = p.downcase
+    @uris ||=
+      destinations.map do |destination|
+        locales.map do |locale|
+          uri = "/#{locale}/".squeeze('/')
+          dir = File.join(destination, locale)
 
-        IMAGES.any? do |i|
-          p.end_with? i
+          next unless File.directory? dir
+
+          files(dir).map do |f|
+            uri + f
+          end
         end
-      end).map do |uri|
-        "/#{uri}"
-      end
+      end.flatten(3).compact
+  end
+
+  # @return [Array]
+  def locales
+    @locales ||= ['', site.locales.map(&:to_s)].flatten(1)
+  end
+
+  # @param :dir [String]
+  # @return [Array]
+  def files(dir)
+    Dir.chdir(dir) do
+      pages = Dir.glob('**/*.html')
+      files = Dir.glob('public/**/*')
+      files = remove_directories files
+      files = remove_images files
+
+      [pages, files].flatten(1)
+    end
+  end
+
+  # @param :files [Array]
+  # @return [Array]
+  def remove_directories(files)
+    files.reject do |f|
+      File.directory? f
+    end
+  end
+
+  def remove_images(files)
+    files.reject do |f|
+      IMAGES.include? File.extname(f).downcase
     end
   end
 end

app/models/site.rb

@@ -101,6 +101,26 @@ class Site < ApplicationRecord
     "https://#{hostname}#{slash ? '/' : ''}"
   end
 
+  # TODO: Cambiar al mergear origin-referer
+  #
+  # @return [Array]
+  def hostnames
+    @hostnames ||= deploys.map do |deploy|
+      case deploy
+      when DeployLocal
+        hostname
+      when DeployWww
+        deploy.fqdn
+      when DeployAlternativeDomain
+        deploy.hostname.dup.tap do |h|
+          h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}")
+        end
+      when DeployHiddenService
+        deploy.onion
+      end
+    end.compact
+  end
+
   # Obtiene los dominios alternativos
   #
   # @return Array
@@ -123,7 +143,9 @@ class Site < ApplicationRecord
   #
   # @return Array
   def urls(slash: true)
-    alternative_urls(slash: slash) << url(slash: slash)
+    @urls ||= hostnames.map do |h|
+      "https://#{h}#{slash ? '/' : ''}"
+    end
   end
 
   def invitade?(usuarie)

app/models/site_blazer.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+SiteBlazer = Struct.new(:site)

app/models/stat.rb

@@ -3,8 +3,16 @@
 # Registran cuándo fue la última recolección de datos.
 class Stat < ApplicationRecord
   # XXX: Los intervalos van en orden de mayor especificidad a menor
-  INTERVALS = %i[day month year].freeze
+  INTERVALS = %i[day].freeze
   RESOURCES = %i[builds space_used build_time].freeze
+  COLUMNS = %i[http_referer geoip2_data_country_name].freeze
 
   belongs_to :site
+
+  # El intervalo por defecto
+  #
+  # @return [Symbol]
+  def self.default_interval
+    INTERVALS.first
+  end
 end

app/models/usuarie.rb

@@ -11,6 +11,8 @@ class Usuarie < ApplicationRecord
 
   has_many :roles
   has_many :sites, through: :roles
+  has_many :blazer_audits, foreign_key: 'user_id', class_name: 'Blazer::Audit'
+  has_many :blazer_queries, foreign_key: 'creator_id', class_name: 'Blazer::Query'
 
   def name
     email.split('@', 2).first

app/policies/site_blazer_policy.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# Les invitades no pueden ver las estadísticas (aun)
+SiteBlazerPolicy = Struct.new(:usuarie, :site_blazer) do
+  def home?
+    site_blazer&.site&.usuarie? usuarie
+  end
+
+  alias_method :show?, :home?
+end

app/views/blazer/check_mailer/failing_checks.haml

@@ -0,0 +1,5 @@
+%ul
+  - @checks.each do |check|
+    %li
+      = check.query.name
+      = check.state

app/views/blazer/check_mailer/state_change.haml

@@ -0,0 +1,30 @@
+!!!
+%html
+  %head
+    %meta{:content => "text/html; charset=UTF-8", "http-equiv" => "Content-Type"}/
+  %body{:style => "font-family: 'Helvetica Neue', Arial, Helvetica; font-size: 14px; color: #333;"}
+    - if @error
+      %p= @error
+    - elsif @rows_count > 0 && @check_type == "bad_data"
+      %p
+        - if @rows_count <= 10
+          = pluralize(@rows_count, "row")
+        - else
+          Showing 10 of #{@rows_count} rows
+      %table{:style => "width: 100%; border-spacing: 0; border-collapse: collapse;"}
+        %thead
+          %tr
+            - @columns.first(5).each do |column|
+              %th{:style => "padding: 8px; line-height: 1.4; text-align: left; vertical-align: bottom; border-bottom: 2px solid #ddd; width: #{(100 / @columns.size).round(2)}%;"}
+                = column
+        %tbody
+          - @rows.first(10).each do |row|
+            %tr
+              - @columns.first(5).each_with_index do |column, i|
+                %td{:style => "padding: 8px; line-height: 1.4; vertical-align: top; border-top: 1px solid #ddd;"}
+                  - value = row[i]
+                  - if @column_types[i] == "time" && value.to_s.length > 10
+                    - value = Time.parse(value).in_time_zone(Blazer.time_zone) rescue value
+                  = value
+      - if @columns.size > 5
+        %p{:style => "color: #999;"} Only first 5 columns shown

app/views/blazer/queries/home.haml

@@ -0,0 +1,9 @@
+#queries
+  %table.table
+    %tbody.list
+      - @queries.each do |query|
+        %tr
+          -#
+            Por alguna razón no tenemos acceso a query_path para poder
+            generar la URL según Rails
+          %td= link_to query[:name], "/sites/#{params[:site_id]}/stats/queries/#{query.to_param}"

app/views/blazer/queries/show.haml

@@ -0,0 +1,51 @@
+- blazer_title @query.name
+.container
+  .row
+    .col-12
+      %h1= @query.name
+      - if @query.description.present?
+        %p.lead= @query.description
+    - unless @result.chart_type.blank?
+      .col-12
+        - case @result.chart_type
+        - when 'line'
+          = line_chart @chart_data, **@chart_options
+        - when 'line2'
+          = line_chart @chart_data, **@chart_options
+        - when 'pie'
+          = pie_chart @chart_data, **@chart_options
+        - when 'bar'
+          = column_chart @chart_data, **@chart_options
+        - when 'bar2'
+          = column_chart @chart_data, **@chart_options
+        - when 'scatter'
+          = scatter_chart @chart_data, **@chart_options
+    .col-12
+      %table.table
+        %thead
+          %tr
+            - @result.columns.each do |key|
+              - next if key.include? 'ciphertext'
+              - next if key.include? 'encrypted'
+              %th.position-sticky.background-white{ style: 'top: 0' }= t("blazer.columns.#{key}", default: key.titleize)
+        %tbody
+          - @result.rows.each do |row|
+            %tr
+              - row.each_with_index do |v, i|
+                - k = @result.columns[i]
+                - next if k.include? 'ciphertext'
+                - next if k.include? 'encrypted'
+                %td
+                  - if v.is_a?(Time)
+                    - v = blazer_time_value(@data_source, k, v)
+
+                  - unless v.nil?
+                    - if v.is_a?(String) && v.empty?
+                      %span.text-muted= t('.empty')
+                    - elsif @data_source.linked_columns[k]
+                      = link_to blazer_format_value(k, v), @data_source.linked_columns[k].gsub('{value}', u(v.to_s)), target: '_blank'
+                    - else
+                      = blazer_format_value(k, v)
+
+                  - if (v2 = (@result.boom[k] || {})[v.nil? ? v : v.to_s])
+                    %span.text-muted= v2

app/views/layouts/_breadcrumb.haml

@@ -12,7 +12,7 @@
           - else
             %span.line-clamp-1= link_to crumb.name, crumb.url
 
-  - if current_usuarie
+  - if @current_usuarie || current_usuarie
     %ul.navbar-nav
       - if @site&.tienda?
         %li.nav-item
@@ -20,5 +20,5 @@
             role: 'button', class: 'btn'
 
       %li.nav-item
-        = link_to t('.logout'), destroy_usuarie_session_path,
+        = link_to t('.logout'), main_app.destroy_usuarie_session_path,
           method: :delete, role: 'button', class: 'btn'

app/views/layouts/blazer/application.haml

@@ -0,0 +1,14 @@
+!!!
+%html
+  %head
+    %meta{content: 'text/html; charset=UTF-8', 'http-equiv': 'Content-Type'}/
+    %title= blazer_title ? blazer_title : 'Sutty'
+    %meta{charset: 'utf-8'}/
+    = favicon_link_tag 'blazer/favicon.png'
+    = stylesheet_link_tag 'application'
+    = javascript_pack_tag 'blazer', 'data-turbolinks-track': 'reload'
+    = csrf_meta_tags
+  %body{ class: yield(:body) }
+    .container-fluid#sutty
+      = render 'layouts/breadcrumb'
+      = yield

app/views/posts/index.haml

@@ -15,6 +15,9 @@
           - else
             %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'btn btn-secondary btn-sm'
 
+    - if policy(@site_stat).index?
+      = link_to t('stats.index.title'), site_stats_path(@site), class: 'btn'
+
     - if policy(@site).edit?
       = link_to t('sites.edit.btn', site: @site.title), edit_site_path(@site), class: 'btn'
 
@@ -71,8 +74,8 @@
         %table.table{ data: { controller: 'reorder' } }
           %caption.sr-only= t('posts.caption')
           %thead
-            %tr
-              %th.border-0.background-white.position-sticky{ style: 'top: 0; z-index: 2', colspan: '4' }
+            %tr.sticky-top
+              %th.border-0{ colspan: '4' }
                 .d-flex.flex-row.justify-content-between
                   %div
                     = submit_tag t('posts.reorder.submit'), class: 'btn'

app/views/stats/index.haml

@@ -6,32 +6,57 @@
       %p
         %small
           = t('.last_update')
-          %time{ datetime: @last_stat.created_at }
-            #{time_ago_in_words @last_stat.created_at}.
+          %time{ datetime: @last_stat.updated_at }
+            #{time_ago_in_words @last_stat.updated_at}.
 
-    .mb-5
+    %form.mb-5.form-inline{ method: 'get' }
       - Stat::INTERVALS.each do |interval|
-        = link_to t(".#{interval}"), site_stats_path(interval: interval, urls: params[:urls]), class: "btn #{'btn-primary active' if @interval == interval}"
+        = link_to t(".#{interval}"), site_stats_path(interval: interval, urls: params[:urls], period_start: params[:period_start].to_date.try(:"beginning_of_#{interval}").to_date, period_end: params[:period_end]), class: "mb-0 btn #{'btn-primary active' if @interval == interval}"
+
+      %input.form-control{ type: 'date', name: :period_start, value: params[:period_start] }
+      %input.form-control{ type: 'date', name: :period_end, value: params[:period_end] }
+      %button.btn.mb-0{ type: 'submit' }= t('.filter')
 
     .mb-5
       %h2= t('.host.title', count: @hostnames.size)
       %p.lead= t('.host.description')
       = line_chart site_stats_host_path(@chart_params), **@chart_options
 
-    .mb-5
+    #custom-urls.mb-5
       %h2= t('.urls.title')
       %p.lead= t('.urls.description')
-      %form
+      %form{ method: 'get', action: '#custom-urls' }
         %input{ type: 'hidden', name: 'interval', value: @interval }
         .form-group
           %label{ for: 'urls' }= t('.urls.label')
-          %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size, aria_describedby: 'help-urls' }= @normalized_urls.join("\n")
+          %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size + 1, aria_describedby: 'help-urls' }= @normalized_urls.join("\n")
           %small#help-urls.feedback.form-text.text-muted= t('.urls.help')
         .form-group
           %button.btn{ type: 'submit' }= t('.urls.submit')
       - if @normalized_urls.present?
-        = line_chart site_stats_uris_path(urls: params[:urls], **@chart_params), **@chart_options
+        = line_chart site_stats_uris_path(urls: @normalized_urls, **@chart_params), **@chart_options
 
+    .row.mb-5.row-cols-1.row-cols-md-2
+      - @columns.each_pair do |column, values|
+        - next if values.blank?
+        .col.mb-5
+          %h2= t(".columns.#{column}.title")
+          %p.lead= t(".columns.#{column}.description")
+
+          %table.table
+            %colgroup
+              %col
+              %col
+            %thead
+              %tr.sticky-top
+                %th{ scope: 'col' }= t(".columns.#{column}.column")
+                %th{ scope: 'col' }= t('.columns.visits')
+            %tfoot
+            %tbody
+              - values.each_pair do |col, val|
+                %tr
+                  %th{ scope: 'row', style: 'word-break: break-all' }= col.blank? ? t(".columns.#{column}.empty") : col
+                  %td= val
     .mb-5
       %h2= t('.resources.title')
       %p.lead= t('.resources.description')

bin/access_logs

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -e
+
+# Volcar y eliminar todos los access logs de dos días atrás
+date="`dateadd today -1d`"
+file="/srv/http/_storage/${date}.psql.gz"
+test -n "${date}"
+test ! -f "${file}"
+
+psql -h postgresql "${DATABASE:-sutty}" sutty <<SQL | gzip > "${file}"
+begin;
+copy (select * from access_logs where created_at < '${date}') to stdout;
+delete from access_logs where created_at < '${date}';
+commit;
+SQL

config/application.rb

@@ -39,6 +39,7 @@ module Sutty
     config.active_storage.variant_processor = :vips
 
     config.to_prepare do
+      # Load application's model / class decorators
       Dir.glob(File.join(File.dirname(__FILE__), '..', 'app', '**', '*_decorator.rb')).sort.each do |c|
         Rails.configuration.cache_classes ? require(c) : load(c)
       end

config/blazer.yml

@@ -50,7 +50,7 @@ user_method: current_usuarie
 user_name: email
 
 # custom before_action to use for auth
-# before_action_method: require_admin
+before_action_method: require_usuarie
 
 # email to send checks from
 from_email: blazer@<%= ENV.fetch('SUTTY', 'sutty.nl') %>

config/locales/en.yml

@@ -258,7 +258,7 @@ en:
       help: |
         These statistics show information about how your site is generated and
         how many resources it uses.
-      last_update: 'Updated every hour.  Last update on '
+      last_update: 'Updated daily.  Last update on '
       empty: 'There is no enough information yet.  We invite you to come back in %{please_return_at}!'
       loading: 'Loading...'
       hour: 'Hourly'
@@ -289,7 +289,19 @@ en:
           description: 'Average storage space used by your site.'
         build_time:
           title: 'Publication time'
-          description: 'Average time your site takes to build.'
+          description: 'Average time your site takes to build, from pressing "Publish changes" to actually being available on your site.'
+      columns:
+        visits: "Visits"
+        http_referer:
+          title: "Referers"
+          description: "Visits by origin"
+          column: "Referer"
+          empty: "(direct visit)"
+        geoip2_data_country_name:
+          title: "Countries"
+          description: "Visits by country"
+          column: "Country"
+          empty: "(couldn't detect country)"
   sites:
     donations:
       url: 'https://donaciones.sutty.nl/en/'
@@ -616,3 +628,14 @@ en:
         edit: 'Editing'
       usuaries:
         index: 'Users'
+      stats:
+        index: 'Statistics'
+  blazer:
+    columns:
+      total: 'Total'
+      dia: 'Date'
+      date: 'Date'
+      visitas: 'Visits'
+    queries:
+      show:
+        empty: '(empty)'

config/locales/es.yml

@@ -263,7 +263,7 @@ es:
       help: |
         Las estadísticas visibilizan información sobre cómo se genera y
         cuántos recursos utiliza tu sitio.
-      last_update: 'Actualizadas cada hora.  Última actualización hace '
+      last_update: 'Actualizadas diariamente.  Última actualización hace '
       empty: 'Todavía no hay información suficiente. Te invitamos a volver en %{please_return_at} :)'
       loading: 'Cargando...'
       hour: 'Por hora'
@@ -294,7 +294,19 @@ es:
           description: 'Espacio en disco que ocupa en promedio tu sitio.'
         build_time:
           title: 'Tiempo de publicación'
-          description: 'Tiempo promedio que toma en publicarse tu sitio.'
+          description: 'Tiempo que tarda el sitio en generarse, desde que usas el botón "Publicar cambios" hasta que los puedes ver en el sitio'
+      columns:
+        visits: "Visitas"
+        http_referer:
+          title: "Referencias"
+          description: "Orígenes de las visitas"
+          column: "Referencia"
+          empty: "(visita directa)"
+        geoip2_data_country_name:
+          title: "Países"
+          description: "Cantidad de visitas por país"
+          column: "País"
+          empty: "(no se pudo detectar el país)"
   sites:
     donations:
       url: 'https://donaciones.sutty.nl/'
@@ -624,3 +636,14 @@ es:
         edit: 'Editando'
       usuaries:
         index: 'Usuaries'
+      stats:
+        index: 'Estadísticas'
+  blazer:
+    columns:
+      total: 'Total'
+      dia: 'Fecha'
+      date: 'Fecha'
+      visitas: 'Visitas'
+    queries:
+      show:
+        empty: '(vacío)'

config/routes.rb

@@ -4,8 +4,6 @@ Rails.application.routes.draw do
   devise_for :usuaries
   get '/.well-known/change-password', to: redirect('/usuaries/edit')
 
-  mount Blazer::Engine, at: 'blazer'
-
   root 'application#index'
 
   constraints(Constraints::ApiSubdomain.new) do

db/migrate/20200206163257_install_blazer.rb

@@ -3,8 +3,6 @@
 # Blazer
 class InstallBlazer < ActiveRecord::Migration[6.0]
   def change
-    return unless Rails.env.production?
-
     create_table :blazer_queries do |t|
       t.references :creator
       t.string :name

db/migrate/20211022224008_add_site_to_stats.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# La recolección de estadísticas podría pertenecer a un sitio
+class AddSiteToStats < ActiveRecord::Migration[6.1]
+  def change
+    add_belongs_to :stats, :site, index: true, null: true
+  end
+end

db/migrate/20211022225449_add_name_to_stats.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+# Agregarle un nombre a la estadística
+class AddNameToStats < ActiveRecord::Migration[6.1]
+  def change
+    add_column :stats, :name, :string, null: false
+    add_index :stats, :name, using: 'hash'
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2021_05_14_165639) do
+ActiveRecord::Schema.define(version: 2021_10_22_225449) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"
@@ -64,6 +64,7 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do
     t.string "remote_user"
     t.boolean "crawler", default: false
     t.string "http_referer"
+    t.datetime "created_at", precision: 6
     t.index ["geoip2_data_city_name"], name: "index_access_logs_on_geoip2_data_city_name"
     t.index ["geoip2_data_country_name"], name: "index_access_logs_on_geoip2_data_country_name"
     t.index ["host"], name: "index_access_logs_on_host"
@@ -303,6 +304,15 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do
     t.index ["usuarie_id"], name: "index_roles_on_usuarie_id"
   end
 
+  create_table "rollups", force: :cascade do |t|
+    t.string "name", null: false
+    t.string "interval", null: false
+    t.datetime "time", null: false
+    t.jsonb "dimensions", default: {}, null: false
+    t.float "value"
+    t.index ["name", "interval", "time", "dimensions"], name: "index_rollups_on_name_and_interval_and_time_and_dimensions", unique: true
+  end
+
   create_table "sites", force: :cascade do |t|
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
@@ -324,6 +334,15 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do
     t.index ["name"], name: "index_sites_on_name", unique: true
   end
 
+  create_table "stats", force: :cascade do |t|
+    t.datetime "created_at", precision: 6, null: false
+    t.datetime "updated_at", precision: 6, null: false
+    t.bigint "site_id"
+    t.string "name", null: false
+    t.index ["name"], name: "index_stats_on_name", using: :hash
+    t.index ["site_id"], name: "index_stats_on_site_id"
+  end
+
   create_table "usuaries", force: :cascade do |t|
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
@@ -370,4 +389,10 @@ new.indexed_content := to_tsvector(('pg_catalog.' || new.dictionary)::regconfig,
     SQL_ACTIONS
   end
 
+  create_trigger("access_logs_before_insert_row_tr", :compatibility => 1).
+      on("access_logs").
+      before(:insert) do
+    "new.created_at := to_timestamp(new.msec);"
+  end
+
 end

lib/tasks/stats.rake

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+namespace :stats do
+  desc 'Process stats'
+  task process_all: :environment do
+    Site.all.pluck(:id).each do |site_id|
+      UriCollectionJob.perform_now site_id: site_id, once: true
+      StatCollectionJob.perform_now site_id: site_id, once: true
+    end
+  end
+end

monit.conf

@@ -25,3 +25,13 @@ check program blazer
   with path "/usr/local/bin/sutty blazer"
   every 61 cycles
   if status != 0 then alert
+
+check program access_logs
+  with path "/srv/http/bin/access_logs" as uid "app" and gid "www-data"
+  every "0 0 * * *"
+  if status != 0 then alert
+
+check program stats
+  with path "/usr/bin/foreman run -f /srv/Procfile -d /srv stats" as uid "rails" gid "www-data"
+  every "0 1 * * *"
+  if status != 0 then alert

yarn.lock

@@ -2257,7 +2257,7 @@ collection-visit@^1.0.0:
     map-visit "^1.0.0"
     object-visit "^1.0.0"
 
-color-convert@^1.9.0, color-convert@^1.9.1:
+color-convert@^1.9.0, color-convert@^1.9.1, color-convert@^1.9.3:
   version "1.9.3"
   resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8"
   integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==
@@ -5029,6 +5029,11 @@ mkdirp@^1.0.3, mkdirp@^1.0.4:
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e"
   integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==
 
+moment@^2.10.2:
+  version "2.29.1"
+  resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3"
+  integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==
+
 move-concurrently@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92"