Sutty
/
panel
mirror of https://0xacab.org/sutty/sutty
5
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'blazer' into 'rails' 

Mostrar estadísticas en el panel de Sutty

Closes #2331, #7530, #7529, #7528, #7527, #7512, #7511, #7510, #7509, #7494, #7493, #7492, #7491, #7408, #7407, #7406, #7405, #7397, #7396, #7395, #7394, #7383, #7382, #7381, #7380, #7170, #7169, #7168, #7167, #7154, #7153, #7152, and #7151

See merge request sutty/sutty!52
This commit is contained in:
fauno 2022-12-28 19:25:14 +00:00
parent cf4c719cdb cacb355b62
commit 9681f15b70
40 changed files with 853 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Gemfile
View File

@ -64,7 +64,7 @@ gem 'rails-i18n'
gem 'rails_warden'
gem 'redis', require: %w[redis redis/connection/hiredis]
gem 'redis-rails'
gem 'rollups', git: 'https://github.com/ankane/rollup.git', branch: 'master'
gem 'rollups', git: 'https://github.com/fauno/rollup.git', branch: 'update'
gem 'rubyzip'
gem 'rugged'
gem 'concurrent-ruby-ext'

22
Gemfile.lock
View File

@ -6,15 +6,6 @@ GIT
rails (>= 3.0)
rake (>= 0.8.7)
GIT
remote: https://github.com/ankane/rollup.git
revision: 0ab6c603450175eb1004f7793e86486943cb9f72
branch: master
specs:
rollups (0.1.3)
activesupport (>= 5.1)
groupdate (>= 5.2)
GIT
remote: https://github.com/fauno/email_address
revision: 536b51f7071b68a55140c0c1726b4cd401d1c04d
@ -24,6 +15,15 @@ GIT
netaddr (>= 2.0.4, < 3)
simpleidn
GIT
remote: https://github.com/fauno/rollup.git
revision: ddbb345aa57e63b4cfdf7557267efa89ba60caac
branch: update
specs:
rollups (0.1.3)
activesupport (>= 5.1)
groupdate (>= 5.2)
GEM
remote: https://gems.sutty.nl/
specs:
@ -214,8 +214,8 @@ GEM
ffi (~> 1.0)
globalid (0.6.0)
activesupport (>= 5.0)
groupdate (5.2.2)
activesupport (>= 5)
groupdate (6.1.0)
activesupport (>= 5.2)
hairtrigger (0.2.24)
activerecord (>= 5.0, < 7)
ruby2ruby (~> 2.4)

1
Procfile
View File

@ -5,3 +5,4 @@ blazer_1h: bundle exec rake blazer:run_checks SCHEDULE="1 hour"
blazer_1d: bundle exec rake blazer:run_checks SCHEDULE="1 day"
blazer: bundle exec rake blazer:send_failing_checks
prometheus: bundle exec prometheus_exporter -b 0.0.0.0 --prefix "sutty_"
stats: bundle exec rake stats:process_all

1
app/assets/stylesheets/application.scss
View File

@ -126,6 +126,7 @@ ol.breadcrumb {
color: var(--foreground);
}
.table tr.sticky-top,
.form-control,
.custom-file-label {
background-color: var(--background);

19
app/controllers/application_controller.rb
View File

@ -3,6 +3,7 @@
# Forma de ingreso a Sutty
class ApplicationController < ActionController::Base
include ExceptionHandler
include Pundit
protect_from_forgery with: :null_session, prepend: true
@ -10,6 +11,7 @@ class ApplicationController < ActionController::Base
before_action :configure_permitted_parameters, if: :devise_controller?
around_action :set_locale
rescue_from Pundit::NilPolicyError, with: :page_not_found
rescue_from ActionController::RoutingError, with: :page_not_found
rescue_from ActionController::ParameterMissing, with: :page_not_found
@ -33,7 +35,7 @@ class ApplicationController < ActionController::Base
def find_site
id = params[:site_id] || params[:id]
unless (site = current_usuarie.sites.find_by_name(id))
unless (site = current_usuarie&.sites&.find_by_name(id))
raise SiteNotFound
end
@ -62,6 +64,21 @@ class ApplicationController < ActionController::Base
render 'application/page_not_found', status: :not_found
end
# Necesario para poder acceder a Blazer. Solo les usuaries de este
# sitio pueden acceder al panel.
def require_usuarie
site = find_site
authorize SiteBlazer.new(site)
# Necesario para los breadcrumbs.
ActionView::Base.include Loaf::ViewExtensions unless ActionView::Base.included_modules.include? Loaf::ViewExtensions
breadcrumb current_usuarie.email, main_app.edit_usuarie_registration_path
breadcrumb 'sites.index', main_app.sites_path, match: :exact
breadcrumb site.title, main_app.site_path(site), match: :exact
breadcrumb 'stats.index', root_path, match: :exact
end
protected
def configure_permitted_parameters

194
app/controllers/concerns/blazer_decorator.rb Normal file
View File

@ -0,0 +1,194 @@
# frozen_string_literal: true
# Modificaciones para Blazer
module BlazerDecorator
# No poder obtener información de la base de datos.
module DisableDatabaseInfo
extend ActiveSupport::Concern
included do
def docs; end
def tables; end
def schema; end
end
end
# Deshabilitar edición de consultas y chequeos.
module DisableEdits
extend ActiveSupport::Concern
included do
def create; end
def update; end
def destroy; end
def run; end
def refresh; end
def cancel; end
end
end
# Blazer hace un gran esfuerzo para ejecutar consultas de forma
# asincrónica pero termina enviándolas por JS.
module RunSync
extend ActiveSupport::Concern
included do
alias_method :original_show, :show
include Blazer::BaseHelper
def show
original_show
options = { user: blazer_user, query: @query, run_id: SecureRandom.uuid, async: false }
@data_source = Blazer.data_sources[@query.data_source]
@result = Blazer::RunStatement.new.perform(@data_source, @statement, options)
chart_data
end
private
# Solo mostrar las consultas de le usuarie
def set_queries(_ = nil)
@queries = (@current_usuarie || current_usuarie).blazer_queries
end
# blazer-2.4.2/app/views/blazer/queries/run.html.erb
def chart_type
case @result.chart_type
when /\Aline(2)?\z/
chart_options.merge! min: nil
when /\Abar(2)?\z/
chart_options.merge! library: { tooltips: { intersect: false, axis: 'x' } }
when 'pie'
chart_options
when 'scatter'
chart_options.merge! library: { tooltips: { intersect: false } }, xtitle: @result.columns[0],
ytitle: @result.columns[1]
when nil
else
if @result.column_types.size == 2
chart_options.merge! library: { tooltips: { intersect: false, axis: 'x' } }
else
chart_options.merge! library: { tooltips: { intersect: false } }
end
end
@result.chart_type
end
def chart_data
@chart_data ||=
case chart_type
when 'line'
@result.columns[1..-1].each_with_index.map do |k, i|
{
name: blazer_series_name(k),
data: @result.rows.map do |r|
[r[0], r[i + 1]]
end,
library: series_library[i]
}
end
when 'line2'
@result.rows.group_by do |r|
v = r[1]
(@result.boom[@result.columns[1]] || {})[v.to_s] || v
end.each_with_index.map do |(name, v), i|
{
name: blazer_series_name(name),
data: v.map do |v2|
[v2[0], v2[2]]
end,
library: series_library[i]
}
end
when 'pie'
@result.rows.map do |r|
[(@result.boom[@result.columns[0]] || {})[r[0].to_s] || r[0], r[1]]
end
when 'bar'
(@result.rows.first.size - 1).times.map do |i|
name = @result.columns[i + 1]
{
name: blazer_series_name(name),
data: @result.rows.first(20).map do |r|
[(@result.boom[@result.columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]]
end
}
end
when 'bar2'
first_20 = @result.rows.group_by { |r| r[0] }.values.first(20).flatten(1)
labels = first_20.map { |r| r[0] }.uniq
series = first_20.map { |r| r[1] }.uniq
labels.each do |l|
series.each do |s|
first_20 << [l, s, 0] unless first_20.find { |r| r[0] == l && r[1] == s }
end
end
first_20.group_by do |r|
v = r[1]
(@result.boom[@result.columns[1]] || {})[v.to_s] || v
end.each_with_index.map do |(name, v), _i|
{
name: blazer_series_name(name),
data: v.sort_by do |r2|
labels.index(r2[0])
end.map do |v2|
v3 = v2[0]
[(@result.boom[@result.columns[0]] || {})[v3.to_s] || v3, v2[2]]
end
}
end
when 'scatter'
@result.rows
end
end
def target_index
@target_index ||= @result.columns.index do |k|
k.downcase == 'target'
end
end
def series_library
@series_library ||= {}.tap do |sl|
if target_index
color = '#109618'
sl[target_index - 1] = {
pointStyle: 'line',
hitRadius: 5,
borderColor: color,
pointBackgroundColor: color,
backgroundColor: color,
pointHoverBackgroundColor: color
}
end
end
end
def chart_options
@chart_options ||= { id: SecureRandom.hex }
end
end
end
end
classes = [Blazer::QueriesController, Blazer::ChecksController, Blazer::DashboardsController]
modules = [BlazerDecorator::DisableDatabaseInfo, BlazerDecorator::DisableEdits]
classes.each do |klass|
modules.each do |modul|
klass.include modul unless klass.included_modules.include? modul
end
end
Blazer::QueriesController.include BlazerDecorator::RunSync

5
app/controllers/posts_controller.rb
View File

@ -2,9 +2,6 @@
# Controlador para artículos
class PostsController < ApplicationController
include Pundit
rescue_from Pundit::NilPolicyError, with: :page_not_found
before_action :authenticate_usuarie!
before_action :service_for_direct_upload, only: %i[new edit]
@ -38,6 +35,8 @@ class PostsController < ApplicationController
# Filtrar los posts que les invitades no pueden ver
@usuarie = site.usuarie? current_usuarie
@site_stat = SiteStat.new(site)
end
def show

2
app/controllers/private_controller.rb
View File

@ -6,8 +6,6 @@ class PrivateController < ApplicationController
# XXX: Permite ejecutar JS
skip_forgery_protection
include Pundit
# Enviar el archivo si existe, agregar una / al final siempre para no
# romper las direcciones relativas.
def show

3
app/controllers/sites_controller.rb
View File

@ -2,9 +2,6 @@
# Controlador de sitios
class SitesController < ApplicationController
include Pundit
rescue_from Pundit::NilPolicyError, with: :page_not_found
before_action :authenticate_usuarie!
breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path

113
app/controllers/stats_controller.rb
View File

@ -8,6 +8,10 @@ class StatsController < ApplicationController
before_action :authenticate_usuarie!
before_action :authorize_stats
breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path
breadcrumb 'sites.index', :sites_path, match: :exact
breadcrumb -> { site.title }, -> { site_posts_path(site, locale: locale) }, match: :exact
EXTRA_OPTIONS = {
builds: {},
space_used: { bytes: true },
@ -20,19 +24,53 @@ class StatsController < ApplicationController
policy.script_src :self, :unsafe_inline
end
# Parámetros por defecto
#
# @return [Hash]
def default_url_options
{ interval: 'day', period_start: Date.today.beginning_of_year, period_end: Date.today }
end
def index
@chart_params = { interval: interval }
breadcrumb I18n.t('stats.index.title'), ''
params.with_defaults! default_url_options
@chart_params = {
interval: interval,
period_start: params[:period_start],
period_end: params[:period_end]
}
hostnames
last_stat
chart_options
normalized_urls
expires_in = Time.now.try(:"end_of_#{Stat.default_interval}") - Time.now
@columns = {}
Stat::COLUMNS.each do |column|
@columns[column] =
Rails.cache.fetch("stats/#{column}/#{site.id}", expires_in: expires_in) do
rollup_scope.where(interval: interval, name: "host|#{column}")
.where_dimensions(host: hostnames)
.group("dimensions->>'#{column}'")
.order('sum(value) desc')
.sum(:value)
.transform_values(&:to_i)
.transform_values do |v|
v * nodes
end
end
end
end
# Genera un gráfico de visitas por dominio asociado a este sitio
def host
return unless stale? [last_stat, hostnames, interval]
return unless stale? [last_stat, hostnames, interval, period]
stats = Rollup.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series|
stats = rollup_scope.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series|
series.each do |serie|
serie[:name] = serie.dig(:dimensions, 'host')
serie[:data].transform_values! do |value|
@ -45,23 +83,20 @@ class StatsController < ApplicationController
end
def resources
return unless stale? [last_stat, interval, resource]
return unless stale? [last_stat, interval, resource, period]
options = {
interval: interval,
dimensions: {
deploy_id: @site.deploys.where(type: 'DeployLocal').pluck(:id).first
}
}
options = { interval: interval, dimensions: { site_id: site.id } }
render json: Rollup.series(resource, **options)
render json: rollup_scope.series(resource, **options)
end
def uris
return unless stale? [last_stat, hostnames, interval, normalized_urls]
return unless stale? [last_stat, hostnames, interval, normalized_urls, period]
options = { host: hostnames, uri: normalized_paths }
stats = Rollup.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series|
# XXX: where_dimensions es más corto pero no aprovecha los índices
# de Rollup
stats = rollup_scope.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series|
series.each do |serie|
serie[:name] = serie[:dimensions].slice('host', 'uri').values.join.sub('/index.html', '/')
serie[:data].transform_values! do |value|
@ -75,34 +110,44 @@ class StatsController < ApplicationController
private
def rollup_scope
Rollup.where(time: period)
end
def last_stat
@last_stat ||= Stat.last
@last_stat ||= site.stats.last
end
def authorize_stats
@site = find_site
authorize SiteStat.new(@site)
authorize SiteStat.new(site)
end
# TODO: Eliminar cuando mergeemos referer-origin
def hostnames
@hostnames ||= [@site.hostname, @site.alternative_hostnames].flatten
@hostnames ||= site.hostnames
end
# Normalizar las URLs
#
# @return [Array]
def normalized_urls
@normalized_urls ||= params.permit(:urls).try(:[],
:urls)&.split("\n")&.map(&:strip)&.select(&:present?)&.select do |uri|
uri.start_with? 'https://'
end&.map do |u|
# XXX: Eliminar
# @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1}
next u unless u.end_with? '/'
@normalized_urls ||=
begin
urls = params[:urls].is_a?(Array) ? params[:urls] : params[:urls]&.split("\n")
urls = urls&.map(&:strip)&.select(&:present?)&.select do |uri|
uri.start_with? 'https://'
end
"#{u}index.html"
end&.uniq || [@site.url, @site.urls].flatten.uniq
urls ||= [site.url]
urls.map do |u|
# XXX: Eliminar al deployear
# @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1}
next u unless u.end_with? '/'
"#{u}index.html"
end.uniq
end
end
def normalized_paths
@ -140,14 +185,15 @@ class StatsController < ApplicationController
def interval
@interval ||= begin
i = params[:interval]&.to_sym
Stat::INTERVALS.include?(i) ? i : :day
Stat::INTERVALS.include?(i) ? i : Stat::INTERVALS.first
end
end
# @return [Symbol]
def resource
@resource ||= begin
r = params[:resource].to_sym
Stat::RESOURCES.include?(r) ? r : :builds
Stat::RESOURCES.include?(r) ? r : Stat::RESOURCES.first
end
end
@ -165,4 +211,15 @@ class StatsController < ApplicationController
def nodes
@nodes ||= ENV.fetch('NODES', 1).to_i
end
def period
@period ||= begin
p = params.permit(:period_start, :period_end)
p[:period_start]..p[:period_end]
end
end
def site
@site ||= find_site
end
end

60
app/jobs/concerns/recursive_rollup.rb Normal file
View File

@ -0,0 +1,60 @@
# frozen_string_literal: true
# Implementa rollups recursivos
module RecursiveRollup
extend ActiveSupport::Concern
included do
private
# Genera un rollup recursivo en base al período anterior y aplica una
# operación.
#
# @param :name [String]
# @param :interval_previous [String]
# @param :interval [String]
# @param :operation [Symbol]
# @param :dimensions [Hash]
# @param :beginning [Time]
# @return [Rollup]
def recursive_rollup(name:, interval_previous:, interval:, dimensions:, beginning:, operation: :sum)
Rollup.where(name: name, interval: interval_previous, dimensions: dimensions)
.where('time >= ?', beginning.try(:"beginning_of_#{interval}"))
.group(*dimensions_to_jsonb_query(dimensions))
.rollup(name, interval: interval, update: true) do |rollup|
rollup.try(operation, :value)
end
end
# Reducir las estadísticas calculadas aplicando un rollup sobre el
# intervalo más amplio.
#
# @param :name [String]
# @param :operation [Symbol]
# @param :dimensions [Hash]
# @return [nil]
def reduce_rollup(name:, dimensions:, operation: :sum)
Stat::INTERVALS.reduce do |previous, current|
recursive_rollup(name: name,
interval_previous: previous,
interval: current,
dimensions: dimensions,
beginning: beginning_of_interval,
operation: operation)
# Devolver el intervalo actual
current
end
nil
end
# @param :dimensions [Hash]
# @return [Array]
def dimensions_to_jsonb_query(dimensions)
dimensions.keys.map do |key|
"dimensions->'#{key}'"
end
end
end
end

8
app/jobs/periodic_job.rb
View File

@ -52,4 +52,12 @@ class PeriodicJob < ApplicationJob
def beginning_of_interval
@beginning_of_interval ||= last_stat.created_at.try(:"beginning_of_#{starting_interval}")
end
def stop_file
@stop_file ||= Rails.root.join('tmp', self.class.to_s.tableize)
end
def stop?
File.exist? stop_file
end
end

43
app/jobs/stat_collection_job.rb
View File

@ -2,11 +2,15 @@
# Genera resúmenes de información para poder mostrar estadísticas y se
# corre regularmente a sí misma.
class StatCollectionJob < ApplicationJob
class StatCollectionJob < PeriodicJob
include RecursiveRollup
STAT_NAME = 'stat_collection_job'
def perform(site_id:, once: true)
@site = Site.find site_id
beginning = beginning_of_interval
stat = site.stats.create! name: STAT_NAME
scope.rollup('builds', **options)
@ -18,44 +22,23 @@ class StatCollectionJob < ApplicationJob
rollup.average(:seconds)
end
# XXX: Es correcto promediar promedios?
Stat::INTERVALS.reduce do |previous, current|
rollup(name: 'builds', interval_previous: previous, interval: current)
rollup(name: 'space_used', interval_previous: previous, interval: current, operation: :average)
rollup(name: 'build_time', interval_previous: previous, interval: current, operation: :average)
dimensions = { site_id: site_id }
current
end
# Registrar que se hicieron todas las recolecciones
site.stats.create! name: STAT_NAME
reduce_rollup(name: 'builds', operation: :sum, dimensions: dimensions)
reduce_rollup(name: 'space_used', operation: :average, dimensions: dimensions)
reduce_rollup(name: 'build_time', operation: :average, dimensions: dimensions)
stat.touch
run_again! unless once
end
private
# Genera un rollup recursivo en base al período anterior y aplica una
# operación.
#
# @return [NilClass]
def rollup(name:, interval_previous:, interval:, operation: :sum)
Rollup.where(name: name, interval: interval_previous)
.where_dimensions(site_id: site.id)
.group("dimensions->'site_id'")
.rollup(name, interval: interval, update: true) do |rollup|
rollup.try(:operation, :value)
end
end
# Los registros a procesar
#
# @return [ActiveRecord::Relation]
def scope
@scope ||= site.build_stats
.jekyll
.where('created_at => ?', beginning_of_interval)
.group(:site_id)
@scope ||= site.build_stats.jekyll.where('build_stats.created_at >= ?', beginning_of_interval).group(:site_id)
end
# Las opciones por defecto
@ -64,4 +47,8 @@ class StatCollectionJob < ApplicationJob
def options
@options ||= { interval: starting_interval, update: true }
end
def stat_name
STAT_NAME
end
end

182
app/jobs/uri_collection_job.rb
View File

@ -13,94 +13,160 @@
class UriCollectionJob < PeriodicJob
# Ignoramos imágenes porque suelen ser demasiadas y no aportan a las
# estadísticas.
IMAGES = %w[.png .jpg .jpeg .gif .webp].freeze
IMAGES = %w[.png .jpg .jpeg .gif .webp .jfif].freeze
STAT_NAME = 'uri_collection_job'
def perform(site_id:, once: true)
@site = Site.find site_id
hostnames.each do |hostname|
uris.each do |uri|
return if File.exist? Rails.root.join('tmp', 'uri_collection_job_stop')
# Obtener el principio del intervalo anterior
beginning_of_interval
# Recordar la última vez que se corrió la tarea
stat = site.stats.create! name: STAT_NAME
# Columnas a agrupar
columns = Stat::COLUMNS.zip([nil]).to_h
AccessLog.where(host: hostname, uri: uri)
.where('created_at >= ?', beginning_of_interval)
.completed_requests
.non_robots
.group(:host, :uri)
.rollup('host|uri', interval: starting_interval, update: true)
# Las URIs son la fuente de verdad de las visitas, porque son las
# que indican las páginas y recursos descargables, el resto son
# imágenes, CSS, JS y tipografías que no nos aportan números
# significativos.
uri_dimensions = { host: site.hostnames, uri: uris }
host_dimensions = { host: site.hostnames }
# Reducir las estadísticas calculadas aplicando un rollup sobre el
# intervalo más amplio.
Stat::INTERVALS.reduce do |previous, current|
Rollup.where(name: 'host|uri', interval: previous)
.where_dimensions(host: hostname, uri: uri)
.group("dimensions->'host'", "dimensions->'uri'")
.rollup('host|uri', interval: current, update: true) do |rollup|
rollup.sum(:value)
end
# Devolver el intervalo actual
current
end
end
# Recorremos todos los hostnames y uris posibles y luego agrupamos
# recursivamente para no tener que recalcular, asumiendo que es más
# rápido buscar en los rollups indexados que en la tabla en bruto.
#
# Los referers solo se agrupan por host.
columns.each_key do |column|
columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column)
end
# Recordar la última vez que se corrió la tarea
site.stats.create! name: STAT_NAME
# Cantidad de visitas por host
rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions)
reduce_rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions)
# Cantidad de visitas por página/recurso
rollup(name: 'host|uri', dimensions: uri_dimensions)
reduce_rollup(name: 'host|uri', dimensions: uri_dimensions)
# Cantidad de visitas host y parámetro
columns.each_pair do |column, values|
column_name = "host|#{column}"
column_dimensions = { host: site.hostnames }
column_dimensions[column] = values
rollup(name: column_name, dimensions: column_dimensions, filter: uri_dimensions)
reduce_rollup(name: column_name, dimensions: column_dimensions)
end
stat.touch
run_again! unless once
end
private
# Generar un rollup de access logs
#
# @param :name [String]
# @param :beginning [Time]
# @param :dimensions [Hash]
# @param :filter [Hash]
# @return [nil]
def rollup(name:, dimensions:, interval: starting_interval, filter: nil)
AccessLog.where(**(filter || dimensions))
.where('created_at >= ?', beginning_of_interval)
.completed_requests
.non_robots
.group(*dimensions.keys)
.rollup(name, interval: interval, update: true)
end
# Generar rollups con el resto de la información
#
# @param :name [String]
# @param :dimensions [Hash]
# @param :filter [Hash]
# @return [nil]
def reduce_rollup(name:, dimensions:, filter: nil)
Stat::INTERVALS.reduce do |_previous, current|
rollup(name: name, dimensions: dimensions, filter: filter, interval: current)
current
end
nil
end
def stat_name
STAT_NAME
end
# Obtiene todas las ubicaciones de archivos
#
# @return [String]
#
# TODO: Cambiar al mergear origin-referer
def destination
@destination ||= site.deploys.find_by(type: 'DeployLocal').destination
end
# TODO: Cambiar al mergear origin-referer
#
# @return [Array]
def hostnames
@hostnames ||= site.deploys.map do |deploy|
case deploy
when DeployLocal
site.hostname
when DeployWww
deploy.fqdn
when DeployAlternativeDomain
deploy.hostname.dup.tap do |h|
h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}")
end
when DeployHiddenService
deploy.onion
end
end.compact
def destinations
@destinations ||= site.deploys.map(&:destination).compact.select do |d|
File.directory?(d)
end.map do |d|
File.realpath(d)
end.uniq
end
# Recolecta todas las URIs menos imágenes
#
# TODO: Para los sitios con DeployLocalizedDomain estamos buscando
# URIs de más.
#
# @return [Array]
def uris
@uris ||= Dir.chdir destination do
(Dir.glob('**/*.html') + Dir.glob('public/**/*').reject do |p|
File.directory? p
end.reject do |p|
p = p.downcase
@uris ||=
destinations.map do |destination|
locales.map do |locale|
uri = "/#{locale}/".squeeze('/')
dir = File.join(destination, locale)
IMAGES.any? do |i|
p.end_with? i
next unless File.directory? dir
files(dir).map do |f|
uri + f
end
end
end).map do |uri|
"/#{uri}"
end
end.flatten(3).compact
end
# @return [Array]
def locales
@locales ||= ['', site.locales.map(&:to_s)].flatten(1)
end
# @param :dir [String]
# @return [Array]
def files(dir)
Dir.chdir(dir) do
pages = Dir.glob('**/*.html')
files = Dir.glob('public/**/*')
files = remove_directories files
files = remove_images files
[pages, files].flatten(1)
end
end
# @param :files [Array]
# @return [Array]
def remove_directories(files)
files.reject do |f|
File.directory? f
end
end
def remove_images(files)
files.reject do |f|
IMAGES.include? File.extname(f).downcase
end
end
end

24
app/models/site.rb
View File

@ -101,6 +101,26 @@ class Site < ApplicationRecord
"https://#{hostname}#{slash ? '/' : ''}"
end
# TODO: Cambiar al mergear origin-referer
#
# @return [Array]
def hostnames
@hostnames ||= deploys.map do |deploy|
case deploy
when DeployLocal
hostname
when DeployWww
deploy.fqdn
when DeployAlternativeDomain
deploy.hostname.dup.tap do |h|
h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}")
end
when DeployHiddenService
deploy.onion
end
end.compact
end
# Obtiene los dominios alternativos
#
# @return Array
@ -123,7 +143,9 @@ class Site < ApplicationRecord
#
# @return Array
def urls(slash: true)
alternative_urls(slash: slash) << url(slash: slash)
@urls ||= hostnames.map do |h|
"https://#{h}#{slash ? '/' : ''}"
end
end
def invitade?(usuarie)

3
app/models/site_blazer.rb Normal file
View File

@ -0,0 +1,3 @@
# frozen_string_literal: true
SiteBlazer = Struct.new(:site)

10
app/models/stat.rb
View File

@ -3,8 +3,16 @@
# Registran cuándo fue la última recolección de datos.
class Stat < ApplicationRecord
# XXX: Los intervalos van en orden de mayor especificidad a menor
INTERVALS = %i[day month year].freeze
INTERVALS = %i[day].freeze
RESOURCES = %i[builds space_used build_time].freeze
COLUMNS = %i[http_referer geoip2_data_country_name].freeze
belongs_to :site
# El intervalo por defecto
#
# @return [Symbol]
def self.default_interval
INTERVALS.first
end
end

2
app/models/usuarie.rb
View File

@ -11,6 +11,8 @@ class Usuarie < ApplicationRecord
has_many :roles
has_many :sites, through: :roles
has_many :blazer_audits, foreign_key: 'user_id', class_name: 'Blazer::Audit'
has_many :blazer_queries, foreign_key: 'creator_id', class_name: 'Blazer::Query'
def name
email.split('@', 2).first

10
app/policies/site_blazer_policy.rb Normal file
View File

@ -0,0 +1,10 @@
# frozen_string_literal: true
# Les invitades no pueden ver las estadísticas (aun)
SiteBlazerPolicy = Struct.new(:usuarie, :site_blazer) do
def home?
site_blazer&.site&.usuarie? usuarie
end
alias_method :show?, :home?
end

5
app/views/blazer/check_mailer/failing_checks.haml Normal file
View File

@ -0,0 +1,5 @@
%ul
- @checks.each do |check|
%li
= check.query.name
= check.state

30
app/views/blazer/check_mailer/state_change.haml Normal file
View File

@ -0,0 +1,30 @@
!!!
%html
%head
%meta{:content => "text/html; charset=UTF-8", "http-equiv" => "Content-Type"}/
%body{:style => "font-family: 'Helvetica Neue', Arial, Helvetica; font-size: 14px; color: #333;"}
- if @error
%p= @error
- elsif @rows_count > 0 && @check_type == "bad_data"
%p
- if @rows_count <= 10
= pluralize(@rows_count, "row")
- else
Showing 10 of #{@rows_count} rows
%table{:style => "width: 100%; border-spacing: 0; border-collapse: collapse;"}
%thead
%tr
- @columns.first(5).each do |column|
%th{:style => "padding: 8px; line-height: 1.4; text-align: left; vertical-align: bottom; border-bottom: 2px solid #ddd; width: #{(100 / @columns.size).round(2)}%;"}
= column
%tbody
- @rows.first(10).each do |row|
%tr
- @columns.first(5).each_with_index do |column, i|
%td{:style => "padding: 8px; line-height: 1.4; vertical-align: top; border-top: 1px solid #ddd;"}
- value = row[i]
- if @column_types[i] == "time" && value.to_s.length > 10
- value = Time.parse(value).in_time_zone(Blazer.time_zone) rescue value
= value
- if @columns.size > 5
%p{:style => "color: #999;"} Only first 5 columns shown

9
app/views/blazer/queries/home.haml Normal file
View File

@ -0,0 +1,9 @@
#queries
%table.table
%tbody.list
- @queries.each do |query|
%tr
-#
Por alguna razón no tenemos acceso a query_path para poder
generar la URL según Rails
%td= link_to query[:name], "/sites/#{params[:site_id]}/stats/queries/#{query.to_param}"

51
app/views/blazer/queries/show.haml Normal file
View File

@ -0,0 +1,51 @@
- blazer_title @query.name
.container
.row
.col-12
%h1= @query.name
- if @query.description.present?
%p.lead= @query.description
- unless @result.chart_type.blank?
.col-12
- case @result.chart_type
- when 'line'
= line_chart @chart_data, **@chart_options
- when 'line2'
= line_chart @chart_data, **@chart_options
- when 'pie'
= pie_chart @chart_data, **@chart_options
- when 'bar'
= column_chart @chart_data, **@chart_options
- when 'bar2'
= column_chart @chart_data, **@chart_options
- when 'scatter'
= scatter_chart @chart_data, **@chart_options
.col-12
%table.table
%thead
%tr
- @result.columns.each do |key|
- next if key.include? 'ciphertext'
- next if key.include? 'encrypted'
%th.position-sticky.background-white{ style: 'top: 0' }= t("blazer.columns.#{key}", default: key.titleize)
%tbody
- @result.rows.each do |row|
%tr
- row.each_with_index do |v, i|
- k = @result.columns[i]
- next if k.include? 'ciphertext'
- next if k.include? 'encrypted'
%td
- if v.is_a?(Time)
- v = blazer_time_value(@data_source, k, v)
- unless v.nil?
- if v.is_a?(String) && v.empty?
%span.text-muted= t('.empty')
- elsif @data_source.linked_columns[k]
= link_to blazer_format_value(k, v), @data_source.linked_columns[k].gsub('{value}', u(v.to_s)), target: '_blank'
- else
= blazer_format_value(k, v)
- if (v2 = (@result.boom[k] || {})[v.nil? ? v : v.to_s])
%span.text-muted= v2

4
app/views/layouts/_breadcrumb.haml
View File

@ -12,7 +12,7 @@
- else
%span.line-clamp-1= link_to crumb.name, crumb.url
- if current_usuarie
- if @current_usuarie || current_usuarie
%ul.navbar-nav
- if @site&.tienda?
%li.nav-item
@ -20,5 +20,5 @@
role: 'button', class: 'btn'
%li.nav-item
= link_to t('.logout'), destroy_usuarie_session_path,
= link_to t('.logout'), main_app.destroy_usuarie_session_path,
method: :delete, role: 'button', class: 'btn'

14
app/views/layouts/blazer/application.haml Normal file
View File

@ -0,0 +1,14 @@
!!!
%html
%head
%meta{content: 'text/html; charset=UTF-8', 'http-equiv': 'Content-Type'}/
%title= blazer_title ? blazer_title : 'Sutty'
%meta{charset: 'utf-8'}/
= favicon_link_tag 'blazer/favicon.png'
= stylesheet_link_tag 'application'
= javascript_pack_tag 'blazer', 'data-turbolinks-track': 'reload'
= csrf_meta_tags
%body{ class: yield(:body) }
.container-fluid#sutty
= render 'layouts/breadcrumb'
= yield

7
app/views/posts/index.haml
View File

@ -15,6 +15,9 @@
- else
%td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'btn btn-secondary btn-sm'
- if policy(@site_stat).index?
= link_to t('stats.index.title'), site_stats_path(@site), class: 'btn'
- if policy(@site).edit?
= link_to t('sites.edit.btn', site: @site.title), edit_site_path(@site), class: 'btn'
@ -71,8 +74,8 @@
%table.table{ data: { controller: 'reorder' } }
%caption.sr-only= t('posts.caption')
%thead
%tr
%th.border-0.background-white.position-sticky{ style: 'top: 0; z-index: 2', colspan: '4' }
%tr.sticky-top
%th.border-0{ colspan: '4' }
.d-flex.flex-row.justify-content-between
%div
= submit_tag t('posts.reorder.submit'), class: 'btn'

41
app/views/stats/index.haml
View File

@ -6,32 +6,57 @@
%p
%small
= t('.last_update')
%time{ datetime: @last_stat.created_at }
#{time_ago_in_words @last_stat.created_at}.
%time{ datetime: @last_stat.updated_at }
#{time_ago_in_words @last_stat.updated_at}.
.mb-5
%form.mb-5.form-inline{ method: 'get' }
- Stat::INTERVALS.each do |interval|
= link_to t(".#{interval}"), site_stats_path(interval: interval, urls: params[:urls]), class: "btn #{'btn-primary active' if @interval == interval}"
= link_to t(".#{interval}"), site_stats_path(interval: interval, urls: params[:urls], period_start: params[:period_start].to_date.try(:"beginning_of_#{interval}").to_date, period_end: params[:period_end]), class: "mb-0 btn #{'btn-primary active' if @interval == interval}"
%input.form-control{ type: 'date', name: :period_start, value: params[:period_start] }
%input.form-control{ type: 'date', name: :period_end, value: params[:period_end] }
%button.btn.mb-0{ type: 'submit' }= t('.filter')
.mb-5
%h2= t('.host.title', count: @hostnames.size)
%p.lead= t('.host.description')
= line_chart site_stats_host_path(@chart_params), **@chart_options
.mb-5
#custom-urls.mb-5
%h2= t('.urls.title')
%p.lead= t('.urls.description')
%form
%form{ method: 'get', action: '#custom-urls' }
%input{ type: 'hidden', name: 'interval', value: @interval }
.form-group
%label{ for: 'urls' }= t('.urls.label')
%textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size, aria_describedby: 'help-urls' }= @normalized_urls.join("\n")
%textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size + 1, aria_describedby: 'help-urls' }= @normalized_urls.join("\n")
%small#help-urls.feedback.form-text.text-muted= t('.urls.help')
.form-group
%button.btn{ type: 'submit' }= t('.urls.submit')
- if @normalized_urls.present?
= line_chart site_stats_uris_path(urls: params[:urls], **@chart_params), **@chart_options
= line_chart site_stats_uris_path(urls: @normalized_urls, **@chart_params), **@chart_options
.row.mb-5.row-cols-1.row-cols-md-2
- @columns.each_pair do |column, values|
- next if values.blank?
.col.mb-5
%h2= t(".columns.#{column}.title")
%p.lead= t(".columns.#{column}.description")
%table.table
%colgroup
%col
%col
%thead
%tr.sticky-top
%th{ scope: 'col' }= t(".columns.#{column}.column")
%th{ scope: 'col' }= t('.columns.visits')
%tfoot
%tbody
- values.each_pair do |col, val|
%tr
%th{ scope: 'row', style: 'word-break: break-all' }= col.blank? ? t(".columns.#{column}.empty") : col
%td= val
.mb-5
%h2= t('.resources.title')
%p.lead= t('.resources.description')

15
bin/access_logs Executable file
View File

@ -0,0 +1,15 @@
#!/bin/sh
set -e
# Volcar y eliminar todos los access logs de dos días atrás
date="`dateadd today -1d`"
file="/srv/http/_storage/${date}.psql.gz"
test -n "${date}"
test ! -f "${file}"
psql -h postgresql "${DATABASE:-sutty}" sutty <<SQL | gzip > "${file}"
begin;
copy (select * from access_logs where created_at < '${date}') to stdout;
delete from access_logs where created_at < '${date}';
commit;
SQL

1
config/application.rb
View File

@ -39,6 +39,7 @@ module Sutty
config.active_storage.variant_processor = :vips
config.to_prepare do
# Load application's model / class decorators
Dir.glob(File.join(File.dirname(__FILE__), '..', 'app', '**', '*_decorator.rb')).sort.each do |c|
Rails.configuration.cache_classes ? require(c) : load(c)
end

2
config/blazer.yml
View File

@ -50,7 +50,7 @@ user_method: current_usuarie
user_name: email
# custom before_action to use for auth
# before_action_method: require_admin
before_action_method: require_usuarie
# email to send checks from
from_email: blazer@<%= ENV.fetch('SUTTY', 'sutty.nl') %>

27
config/locales/en.yml
View File

@ -258,7 +258,7 @@ en:
help: |
These statistics show information about how your site is generated and
how many resources it uses.
last_update: 'Updated every hour. Last update on '
last_update: 'Updated daily. Last update on '
empty: 'There is no enough information yet. We invite you to come back in %{please_return_at}!'
loading: 'Loading...'
hour: 'Hourly'
@ -289,7 +289,19 @@ en:
description: 'Average storage space used by your site.'
build_time:
title: 'Publication time'
description: 'Average time your site takes to build.'
description: 'Average time your site takes to build, from pressing "Publish changes" to actually being available on your site.'
columns:
visits: "Visits"
http_referer:
title: "Referers"
description: "Visits by origin"
column: "Referer"
empty: "(direct visit)"
geoip2_data_country_name:
title: "Countries"
description: "Visits by country"
column: "Country"
empty: "(couldn't detect country)"
sites:
donations:
url: 'https://donaciones.sutty.nl/en/'
@ -616,3 +628,14 @@ en:
edit: 'Editing'
usuaries:
index: 'Users'
stats:
index: 'Statistics'
blazer:
columns:
total: 'Total'
dia: 'Date'
date: 'Date'
visitas: 'Visits'
queries:
show:
empty: '(empty)'

27
config/locales/es.yml
View File

@ -263,7 +263,7 @@ es:
help: |
Las estadísticas visibilizan información sobre cómo se genera y
cuántos recursos utiliza tu sitio.
last_update: 'Actualizadas cada hora. Última actualización hace '
last_update: 'Actualizadas diariamente. Última actualización hace '
empty: 'Todavía no hay información suficiente. Te invitamos a volver en %{please_return_at} :)'
loading: 'Cargando...'
hour: 'Por hora'
@ -294,7 +294,19 @@ es:
description: 'Espacio en disco que ocupa en promedio tu sitio.'
build_time:
title: 'Tiempo de publicación'
description: 'Tiempo promedio que toma en publicarse tu sitio.'
description: 'Tiempo que tarda el sitio en generarse, desde que usas el botón "Publicar cambios" hasta que los puedes ver en el sitio'
columns:
visits: "Visitas"
http_referer:
title: "Referencias"
description: "Orígenes de las visitas"
column: "Referencia"
empty: "(visita directa)"
geoip2_data_country_name:
title: "Países"
description: "Cantidad de visitas por país"
column: "País"
empty: "(no se pudo detectar el país)"
sites:
donations:
url: 'https://donaciones.sutty.nl/'
@ -624,3 +636,14 @@ es:
edit: 'Editando'
usuaries:
index: 'Usuaries'
stats:
index: 'Estadísticas'
blazer:
columns:
total: 'Total'
dia: 'Fecha'
date: 'Fecha'
visitas: 'Visitas'
queries:
show:
empty: '(vacío)'

2
config/routes.rb
View File

@ -4,8 +4,6 @@ Rails.application.routes.draw do
devise_for :usuaries
get '/.well-known/change-password', to: redirect('/usuaries/edit')
mount Blazer::Engine, at: 'blazer'
root 'application#index'
constraints(Constraints::ApiSubdomain.new) do

2
db/migrate/20200206163257_install_blazer.rb
View File

@ -3,8 +3,6 @@
# Blazer
class InstallBlazer < ActiveRecord::Migration[6.0]
def change
return unless Rails.env.production?
create_table :blazer_queries do |t|
t.references :creator
t.string :name

8
db/migrate/20211022224008_add_site_to_stats.rb Normal file
View File

@ -0,0 +1,8 @@
# frozen_string_literal: true
# La recolección de estadísticas podría pertenecer a un sitio
class AddSiteToStats < ActiveRecord::Migration[6.1]
def change
add_belongs_to :stats, :site, index: true, null: true
end
end

9
db/migrate/20211022225449_add_name_to_stats.rb Normal file
View File

@ -0,0 +1,9 @@
# frozen_string_literal: true
# Agregarle un nombre a la estadística
class AddNameToStats < ActiveRecord::Migration[6.1]
def change
add_column :stats, :name, :string, null: false
add_index :stats, :name, using: 'hash'
end
end

27
db/schema.rb
View File

@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2021_05_14_165639) do
ActiveRecord::Schema.define(version: 2021_10_22_225449) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_trgm"
@ -64,6 +64,7 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do
t.string "remote_user"
t.boolean "crawler", default: false
t.string "http_referer"
t.datetime "created_at", precision: 6
t.index ["geoip2_data_city_name"], name: "index_access_logs_on_geoip2_data_city_name"
t.index ["geoip2_data_country_name"], name: "index_access_logs_on_geoip2_data_country_name"
t.index ["host"], name: "index_access_logs_on_host"
@ -303,6 +304,15 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do
t.index ["usuarie_id"], name: "index_roles_on_usuarie_id"
end
create_table "rollups", force: :cascade do |t|
t.string "name", null: false
t.string "interval", null: false
t.datetime "time", null: false
t.jsonb "dimensions", default: {}, null: false
t.float "value"
t.index ["name", "interval", "time", "dimensions"], name: "index_rollups_on_name_and_interval_and_time_and_dimensions", unique: true
end
create_table "sites", force: :cascade do |t|
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
@ -324,6 +334,15 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do
t.index ["name"], name: "index_sites_on_name", unique: true
end
create_table "stats", force: :cascade do |t|
t.datetime "created_at", precision: 6, null: false
t.datetime "updated_at", precision: 6, null: false
t.bigint "site_id"
t.string "name", null: false
t.index ["name"], name: "index_stats_on_name", using: :hash
t.index ["site_id"], name: "index_stats_on_site_id"
end
create_table "usuaries", force: :cascade do |t|
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
@ -370,4 +389,10 @@ new.indexed_content := to_tsvector(('pg_catalog.' || new.dictionary)::regconfig,
SQL_ACTIONS
end
create_trigger("access_logs_before_insert_row_tr", :compatibility => 1).
on("access_logs").
before(:insert) do
"new.created_at := to_timestamp(new.msec);"
end
end

11
lib/tasks/stats.rake Normal file
View File

@ -0,0 +1,11 @@
# frozen_string_literal: true
namespace :stats do
desc 'Process stats'
task process_all: :environment do
Site.all.pluck(:id).each do |site_id|
UriCollectionJob.perform_now site_id: site_id, once: true
StatCollectionJob.perform_now site_id: site_id, once: true
end
end
end

10
monit.conf
View File

@ -25,3 +25,13 @@ check program blazer
with path "/usr/local/bin/sutty blazer"
every 61 cycles
if status != 0 then alert
check program access_logs
with path "/srv/http/bin/access_logs" as uid "app" and gid "www-data"
every "0 0 * * *"
if status != 0 then alert
check program stats
with path "/usr/bin/foreman run -f /srv/Procfile -d /srv stats" as uid "rails" gid "www-data"
every "0 1 * * *"
if status != 0 then alert

7
yarn.lock
View File

@ -2257,7 +2257,7 @@ collection-visit@^1.0.0:
map-visit "^1.0.0"
object-visit "^1.0.0"
color-convert@^1.9.0, color-convert@^1.9.1:
color-convert@^1.9.0, color-convert@^1.9.1, color-convert@^1.9.3:
version "1.9.3"
resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8"
integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==
@ -5029,6 +5029,11 @@ mkdirp@^1.0.3, mkdirp@^1.0.4:
resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e"
integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==
moment@^2.10.2:
version "2.29.1"
resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3"
integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==
move-concurrently@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92"