diff --git a/Gemfile b/Gemfile index a7482fb7..09471fd2 100644 --- a/Gemfile +++ b/Gemfile @@ -66,7 +66,7 @@ gem 'rails-i18n' gem 'rails_warden' gem 'redis', '~> 4.0', require: %w[redis redis/connection/hiredis] gem 'redis-rails' -gem 'rollups', git: 'https://github.com/ankane/rollup.git', branch: 'master' +gem 'rollups', git: 'https://github.com/fauno/rollup.git', branch: 'update' gem 'rubyzip' gem 'rugged' gem 'concurrent-ruby-ext' diff --git a/Gemfile.lock b/Gemfile.lock index e26b4cc8..34f64920 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,15 +6,6 @@ GIT rails (>= 3.0) rake (>= 0.8.7) -GIT - remote: https://github.com/ankane/rollup.git - revision: f3fbff7e11e4904b3aef8635ee02d84a70348222 - branch: master - specs: - rollups (0.2.0) - activesupport (>= 5.2) - groupdate (>= 6.1) - GIT remote: https://github.com/fauno/email_address revision: 536b51f7071b68a55140c0c1726b4cd401d1c04d @@ -24,6 +15,15 @@ GIT netaddr (>= 2.0.4, < 3) simpleidn +GIT + remote: https://github.com/fauno/rollup.git + revision: ddbb345aa57e63b4cfdf7557267efa89ba60caac + branch: update + specs: + rollups (0.1.3) + activesupport (>= 5.1) + groupdate (>= 5.2) + GEM remote: https://17.3.alpine.gems.sutty.nl/ specs: @@ -221,8 +221,8 @@ GEM activesupport (>= 5.0) groupdate (6.1.0) activesupport (>= 5.2) - hairtrigger (1.0.0) - activerecord (>= 6.0, < 8) + hairtrigger (0.2.24) + activerecord (>= 5.0, < 7) ruby2ruby (~> 2.4) ruby_parser (~> 3.10) haml (6.1.1-x86_64-linux-musl) diff --git a/Procfile b/Procfile index b308ffd5..8f6c7741 100644 --- a/Procfile +++ b/Procfile @@ -5,3 +5,4 @@ blazer_1h: bundle exec rake blazer:run_checks SCHEDULE="1 hour" blazer_1d: bundle exec rake blazer:run_checks SCHEDULE="1 day" blazer: bundle exec rake blazer:send_failing_checks prometheus: bundle exec prometheus_exporter -b 0.0.0.0 --prefix "sutty_" +stats: bundle exec rake stats:process_all diff --git a/app/assets/stylesheets/application.scss b/app/assets/stylesheets/application.scss index e806a032..b756759a 100644 --- a/app/assets/stylesheets/application.scss +++ b/app/assets/stylesheets/application.scss @@ -126,6 +126,7 @@ ol.breadcrumb { color: var(--foreground); } +.table tr.sticky-top, .form-control, .custom-file-label { background-color: var(--background); diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index acd0134d..d8498218 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -3,6 +3,7 @@ # Forma de ingreso a Sutty class ApplicationController < ActionController::Base include ExceptionHandler + include Pundit protect_from_forgery with: :null_session, prepend: true @@ -10,6 +11,7 @@ class ApplicationController < ActionController::Base before_action :configure_permitted_parameters, if: :devise_controller? around_action :set_locale + rescue_from Pundit::NilPolicyError, with: :page_not_found rescue_from ActionController::RoutingError, with: :page_not_found rescue_from ActionController::ParameterMissing, with: :page_not_found @@ -33,7 +35,7 @@ class ApplicationController < ActionController::Base def find_site id = params[:site_id] || params[:id] - unless (site = current_usuarie.sites.find_by_name(id)) + unless (site = current_usuarie&.sites&.find_by_name(id)) raise SiteNotFound end @@ -62,6 +64,21 @@ class ApplicationController < ActionController::Base render 'application/page_not_found', status: :not_found end + # Necesario para poder acceder a Blazer. Solo les usuaries de este + # sitio pueden acceder al panel. + def require_usuarie + site = find_site + authorize SiteBlazer.new(site) + + # Necesario para los breadcrumbs. + ActionView::Base.include Loaf::ViewExtensions unless ActionView::Base.included_modules.include? Loaf::ViewExtensions + + breadcrumb current_usuarie.email, main_app.edit_usuarie_registration_path + breadcrumb 'sites.index', main_app.sites_path, match: :exact + breadcrumb site.title, main_app.site_path(site), match: :exact + breadcrumb 'stats.index', root_path, match: :exact + end + protected def configure_permitted_parameters diff --git a/app/controllers/concerns/blazer_decorator.rb b/app/controllers/concerns/blazer_decorator.rb new file mode 100644 index 00000000..876f423d --- /dev/null +++ b/app/controllers/concerns/blazer_decorator.rb @@ -0,0 +1,194 @@ +# frozen_string_literal: true + +# Modificaciones para Blazer +module BlazerDecorator + # No poder obtener información de la base de datos. + module DisableDatabaseInfo + extend ActiveSupport::Concern + + included do + def docs; end + + def tables; end + + def schema; end + end + end + + # Deshabilitar edición de consultas y chequeos. + module DisableEdits + extend ActiveSupport::Concern + + included do + def create; end + + def update; end + + def destroy; end + + def run; end + + def refresh; end + + def cancel; end + end + end + + # Blazer hace un gran esfuerzo para ejecutar consultas de forma + # asincrónica pero termina enviándolas por JS. + module RunSync + extend ActiveSupport::Concern + + included do + alias_method :original_show, :show + + include Blazer::BaseHelper + + def show + original_show + + options = { user: blazer_user, query: @query, run_id: SecureRandom.uuid, async: false } + @data_source = Blazer.data_sources[@query.data_source] + @result = Blazer::RunStatement.new.perform(@data_source, @statement, options) + chart_data + end + + private + + # Solo mostrar las consultas de le usuarie + def set_queries(_ = nil) + @queries = (@current_usuarie || current_usuarie).blazer_queries + end + + # blazer-2.4.2/app/views/blazer/queries/run.html.erb + def chart_type + case @result.chart_type + when /\Aline(2)?\z/ + chart_options.merge! min: nil + when /\Abar(2)?\z/ + chart_options.merge! library: { tooltips: { intersect: false, axis: 'x' } } + when 'pie' + chart_options + when 'scatter' + chart_options.merge! library: { tooltips: { intersect: false } }, xtitle: @result.columns[0], + ytitle: @result.columns[1] + when nil + else + if @result.column_types.size == 2 + chart_options.merge! library: { tooltips: { intersect: false, axis: 'x' } } + else + chart_options.merge! library: { tooltips: { intersect: false } } + end + end + + @result.chart_type + end + + def chart_data + @chart_data ||= + case chart_type + when 'line' + @result.columns[1..-1].each_with_index.map do |k, i| + { + name: blazer_series_name(k), + data: @result.rows.map do |r| + [r[0], r[i + 1]] + end, + library: series_library[i] + } + end + when 'line2' + @result.rows.group_by do |r| + v = r[1] + (@result.boom[@result.columns[1]] || {})[v.to_s] || v + end.each_with_index.map do |(name, v), i| + { + name: blazer_series_name(name), + data: v.map do |v2| + [v2[0], v2[2]] + end, + library: series_library[i] + } + end + when 'pie' + @result.rows.map do |r| + [(@result.boom[@result.columns[0]] || {})[r[0].to_s] || r[0], r[1]] + end + when 'bar' + (@result.rows.first.size - 1).times.map do |i| + name = @result.columns[i + 1] + + { + name: blazer_series_name(name), + data: @result.rows.first(20).map do |r| + [(@result.boom[@result.columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]] + end + } + end + when 'bar2' + first_20 = @result.rows.group_by { |r| r[0] }.values.first(20).flatten(1) + labels = first_20.map { |r| r[0] }.uniq + series = first_20.map { |r| r[1] }.uniq + labels.each do |l| + series.each do |s| + first_20 << [l, s, 0] unless first_20.find { |r| r[0] == l && r[1] == s } + end + end + + first_20.group_by do |r| + v = r[1] + (@result.boom[@result.columns[1]] || {})[v.to_s] || v + end.each_with_index.map do |(name, v), _i| + { + name: blazer_series_name(name), + data: v.sort_by do |r2| + labels.index(r2[0]) + end.map do |v2| + v3 = v2[0] + [(@result.boom[@result.columns[0]] || {})[v3.to_s] || v3, v2[2]] + end + } + end + when 'scatter' + @result.rows + end + end + + def target_index + @target_index ||= @result.columns.index do |k| + k.downcase == 'target' + end + end + + def series_library + @series_library ||= {}.tap do |sl| + if target_index + color = '#109618' + sl[target_index - 1] = { + pointStyle: 'line', + hitRadius: 5, + borderColor: color, + pointBackgroundColor: color, + backgroundColor: color, + pointHoverBackgroundColor: color + } + end + end + end + + def chart_options + @chart_options ||= { id: SecureRandom.hex } + end + end + end +end + +classes = [Blazer::QueriesController, Blazer::ChecksController, Blazer::DashboardsController] +modules = [BlazerDecorator::DisableDatabaseInfo, BlazerDecorator::DisableEdits] +classes.each do |klass| + modules.each do |modul| + klass.include modul unless klass.included_modules.include? modul + end +end + +Blazer::QueriesController.include BlazerDecorator::RunSync diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index 2aff9ac9..c5dc0f54 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -2,9 +2,6 @@ # Controlador para artículos class PostsController < ApplicationController - include Pundit - rescue_from Pundit::NilPolicyError, with: :page_not_found - before_action :authenticate_usuarie! before_action :service_for_direct_upload, only: %i[new edit] @@ -38,6 +35,8 @@ class PostsController < ApplicationController # Filtrar los posts que les invitades no pueden ver @usuarie = site.usuarie? current_usuarie + + @site_stat = SiteStat.new(site) end def show diff --git a/app/controllers/private_controller.rb b/app/controllers/private_controller.rb index bb4d782d..01b6888c 100644 --- a/app/controllers/private_controller.rb +++ b/app/controllers/private_controller.rb @@ -6,8 +6,6 @@ class PrivateController < ApplicationController # XXX: Permite ejecutar JS skip_forgery_protection - include Pundit - # Enviar el archivo si existe, agregar una / al final siempre para no # romper las direcciones relativas. def show diff --git a/app/controllers/sites_controller.rb b/app/controllers/sites_controller.rb index bdaa9011..b4826226 100644 --- a/app/controllers/sites_controller.rb +++ b/app/controllers/sites_controller.rb @@ -2,9 +2,6 @@ # Controlador de sitios class SitesController < ApplicationController - include Pundit - rescue_from Pundit::NilPolicyError, with: :page_not_found - before_action :authenticate_usuarie! breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 44073c1f..c2c7bc58 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -8,6 +8,10 @@ class StatsController < ApplicationController before_action :authenticate_usuarie! before_action :authorize_stats + breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path + breadcrumb 'sites.index', :sites_path, match: :exact + breadcrumb -> { site.title }, -> { site_posts_path(site, locale: locale) }, match: :exact + EXTRA_OPTIONS = { builds: {}, space_used: { bytes: true }, @@ -20,19 +24,53 @@ class StatsController < ApplicationController policy.script_src :self, :unsafe_inline end + # Parámetros por defecto + # + # @return [Hash] + def default_url_options + { interval: 'day', period_start: Date.today.beginning_of_year, period_end: Date.today } + end + def index - @chart_params = { interval: interval } + breadcrumb I18n.t('stats.index.title'), '' + + params.with_defaults! default_url_options + + @chart_params = { + interval: interval, + period_start: params[:period_start], + period_end: params[:period_end] + } + hostnames last_stat chart_options normalized_urls + + expires_in = Time.now.try(:"end_of_#{Stat.default_interval}") - Time.now + @columns = {} + + Stat::COLUMNS.each do |column| + @columns[column] = + Rails.cache.fetch("stats/#{column}/#{site.id}", expires_in: expires_in) do + rollup_scope.where(interval: interval, name: "host|#{column}") + .where_dimensions(host: hostnames) + .group("dimensions->>'#{column}'") + .order('sum(value) desc') + .sum(:value) + .transform_values(&:to_i) + .transform_values do |v| + v * nodes + end + end + end end # Genera un gráfico de visitas por dominio asociado a este sitio def host - return unless stale? [last_stat, hostnames, interval] + return unless stale? [last_stat, hostnames, interval, period] - stats = Rollup.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series| + stats = rollup_scope.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series| series.each do |serie| serie[:name] = serie.dig(:dimensions, 'host') serie[:data].transform_values! do |value| @@ -45,23 +83,20 @@ class StatsController < ApplicationController end def resources - return unless stale? [last_stat, interval, resource] + return unless stale? [last_stat, interval, resource, period] - options = { - interval: interval, - dimensions: { - deploy_id: @site.deploys.where(type: 'DeployLocal').pluck(:id).first - } - } + options = { interval: interval, dimensions: { site_id: site.id } } - render json: Rollup.series(resource, **options) + render json: rollup_scope.series(resource, **options) end def uris - return unless stale? [last_stat, hostnames, interval, normalized_urls] + return unless stale? [last_stat, hostnames, interval, normalized_urls, period] options = { host: hostnames, uri: normalized_paths } - stats = Rollup.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series| + # XXX: where_dimensions es más corto pero no aprovecha los índices + # de Rollup + stats = rollup_scope.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series| series.each do |serie| serie[:name] = serie[:dimensions].slice('host', 'uri').values.join.sub('/index.html', '/') serie[:data].transform_values! do |value| @@ -75,34 +110,44 @@ class StatsController < ApplicationController private + def rollup_scope + Rollup.where(time: period) + end + def last_stat - @last_stat ||= Stat.last + @last_stat ||= site.stats.last end def authorize_stats - @site = find_site - authorize SiteStat.new(@site) + authorize SiteStat.new(site) end # TODO: Eliminar cuando mergeemos referer-origin def hostnames - @hostnames ||= [@site.hostname, @site.alternative_hostnames].flatten + @hostnames ||= site.hostnames end # Normalizar las URLs # # @return [Array] def normalized_urls - @normalized_urls ||= params.permit(:urls).try(:[], - :urls)&.split("\n")&.map(&:strip)&.select(&:present?)&.select do |uri| - uri.start_with? 'https://' - end&.map do |u| - # XXX: Eliminar - # @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1} - next u unless u.end_with? '/' + @normalized_urls ||= + begin + urls = params[:urls].is_a?(Array) ? params[:urls] : params[:urls]&.split("\n") + urls = urls&.map(&:strip)&.select(&:present?)&.select do |uri| + uri.start_with? 'https://' + end - "#{u}index.html" - end&.uniq || [@site.url, @site.urls].flatten.uniq + urls ||= [site.url] + + urls.map do |u| + # XXX: Eliminar al deployear + # @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1} + next u unless u.end_with? '/' + + "#{u}index.html" + end.uniq + end end def normalized_paths @@ -140,14 +185,15 @@ class StatsController < ApplicationController def interval @interval ||= begin i = params[:interval]&.to_sym - Stat::INTERVALS.include?(i) ? i : :day + Stat::INTERVALS.include?(i) ? i : Stat::INTERVALS.first end end + # @return [Symbol] def resource @resource ||= begin r = params[:resource].to_sym - Stat::RESOURCES.include?(r) ? r : :builds + Stat::RESOURCES.include?(r) ? r : Stat::RESOURCES.first end end @@ -165,4 +211,15 @@ class StatsController < ApplicationController def nodes @nodes ||= ENV.fetch('NODES', 1).to_i end + + def period + @period ||= begin + p = params.permit(:period_start, :period_end) + p[:period_start]..p[:period_end] + end + end + + def site + @site ||= find_site + end end diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb new file mode 100644 index 00000000..1da14c01 --- /dev/null +++ b/app/jobs/concerns/recursive_rollup.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +# Implementa rollups recursivos +module RecursiveRollup + extend ActiveSupport::Concern + + included do + private + + # Genera un rollup recursivo en base al período anterior y aplica una + # operación. + # + # @param :name [String] + # @param :interval_previous [String] + # @param :interval [String] + # @param :operation [Symbol] + # @param :dimensions [Hash] + # @param :beginning [Time] + # @return [Rollup] + def recursive_rollup(name:, interval_previous:, interval:, dimensions:, beginning:, operation: :sum) + Rollup.where(name: name, interval: interval_previous, dimensions: dimensions) + .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) + .group(*dimensions_to_jsonb_query(dimensions)) + .rollup(name, interval: interval, update: true) do |rollup| + rollup.try(operation, :value) + end + end + + # Reducir las estadísticas calculadas aplicando un rollup sobre el + # intervalo más amplio. + # + # @param :name [String] + # @param :operation [Symbol] + # @param :dimensions [Hash] + # @return [nil] + def reduce_rollup(name:, dimensions:, operation: :sum) + Stat::INTERVALS.reduce do |previous, current| + recursive_rollup(name: name, + interval_previous: previous, + interval: current, + dimensions: dimensions, + beginning: beginning_of_interval, + operation: operation) + + # Devolver el intervalo actual + current + end + + nil + end + + # @param :dimensions [Hash] + # @return [Array] + def dimensions_to_jsonb_query(dimensions) + dimensions.keys.map do |key| + "dimensions->'#{key}'" + end + end + end +end diff --git a/app/jobs/periodic_job.rb b/app/jobs/periodic_job.rb index 8d9453a3..2f60a2b3 100644 --- a/app/jobs/periodic_job.rb +++ b/app/jobs/periodic_job.rb @@ -52,4 +52,12 @@ class PeriodicJob < ApplicationJob def beginning_of_interval @beginning_of_interval ||= last_stat.created_at.try(:"beginning_of_#{starting_interval}") end + + def stop_file + @stop_file ||= Rails.root.join('tmp', self.class.to_s.tableize) + end + + def stop? + File.exist? stop_file + end end diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 2aa8d702..e402e3b5 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -2,11 +2,15 @@ # Genera resúmenes de información para poder mostrar estadísticas y se # corre regularmente a sí misma. -class StatCollectionJob < ApplicationJob +class StatCollectionJob < PeriodicJob + include RecursiveRollup + STAT_NAME = 'stat_collection_job' def perform(site_id:, once: true) @site = Site.find site_id + beginning = beginning_of_interval + stat = site.stats.create! name: STAT_NAME scope.rollup('builds', **options) @@ -18,44 +22,23 @@ class StatCollectionJob < ApplicationJob rollup.average(:seconds) end - # XXX: Es correcto promediar promedios? - Stat::INTERVALS.reduce do |previous, current| - rollup(name: 'builds', interval_previous: previous, interval: current) - rollup(name: 'space_used', interval_previous: previous, interval: current, operation: :average) - rollup(name: 'build_time', interval_previous: previous, interval: current, operation: :average) + dimensions = { site_id: site_id } - current - end - - # Registrar que se hicieron todas las recolecciones - site.stats.create! name: STAT_NAME + reduce_rollup(name: 'builds', operation: :sum, dimensions: dimensions) + reduce_rollup(name: 'space_used', operation: :average, dimensions: dimensions) + reduce_rollup(name: 'build_time', operation: :average, dimensions: dimensions) + stat.touch run_again! unless once end private - # Genera un rollup recursivo en base al período anterior y aplica una - # operación. - # - # @return [NilClass] - def rollup(name:, interval_previous:, interval:, operation: :sum) - Rollup.where(name: name, interval: interval_previous) - .where_dimensions(site_id: site.id) - .group("dimensions->'site_id'") - .rollup(name, interval: interval, update: true) do |rollup| - rollup.try(:operation, :value) - end - end - # Los registros a procesar # # @return [ActiveRecord::Relation] def scope - @scope ||= site.build_stats - .jekyll - .where('created_at => ?', beginning_of_interval) - .group(:site_id) + @scope ||= site.build_stats.jekyll.where('build_stats.created_at >= ?', beginning_of_interval).group(:site_id) end # Las opciones por defecto @@ -64,4 +47,8 @@ class StatCollectionJob < ApplicationJob def options @options ||= { interval: starting_interval, update: true } end + + def stat_name + STAT_NAME + end end diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 9ec333cd..4cbbf593 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -13,94 +13,160 @@ class UriCollectionJob < PeriodicJob # Ignoramos imágenes porque suelen ser demasiadas y no aportan a las # estadísticas. - IMAGES = %w[.png .jpg .jpeg .gif .webp].freeze + IMAGES = %w[.png .jpg .jpeg .gif .webp .jfif].freeze STAT_NAME = 'uri_collection_job' def perform(site_id:, once: true) @site = Site.find site_id - hostnames.each do |hostname| - uris.each do |uri| - return if File.exist? Rails.root.join('tmp', 'uri_collection_job_stop') + # Obtener el principio del intervalo anterior + beginning_of_interval + # Recordar la última vez que se corrió la tarea + stat = site.stats.create! name: STAT_NAME + # Columnas a agrupar + columns = Stat::COLUMNS.zip([nil]).to_h - AccessLog.where(host: hostname, uri: uri) - .where('created_at >= ?', beginning_of_interval) - .completed_requests - .non_robots - .group(:host, :uri) - .rollup('host|uri', interval: starting_interval, update: true) + # Las URIs son la fuente de verdad de las visitas, porque son las + # que indican las páginas y recursos descargables, el resto son + # imágenes, CSS, JS y tipografías que no nos aportan números + # significativos. + uri_dimensions = { host: site.hostnames, uri: uris } + host_dimensions = { host: site.hostnames } - # Reducir las estadísticas calculadas aplicando un rollup sobre el - # intervalo más amplio. - Stat::INTERVALS.reduce do |previous, current| - Rollup.where(name: 'host|uri', interval: previous) - .where_dimensions(host: hostname, uri: uri) - .group("dimensions->'host'", "dimensions->'uri'") - .rollup('host|uri', interval: current, update: true) do |rollup| - rollup.sum(:value) - end - - # Devolver el intervalo actual - current - end - end + # Recorremos todos los hostnames y uris posibles y luego agrupamos + # recursivamente para no tener que recalcular, asumiendo que es más + # rápido buscar en los rollups indexados que en la tabla en bruto. + # + # Los referers solo se agrupan por host. + columns.each_key do |column| + columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) end - # Recordar la última vez que se corrió la tarea - site.stats.create! name: STAT_NAME + # Cantidad de visitas por host + rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) + reduce_rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) + + # Cantidad de visitas por página/recurso + rollup(name: 'host|uri', dimensions: uri_dimensions) + reduce_rollup(name: 'host|uri', dimensions: uri_dimensions) + + # Cantidad de visitas host y parámetro + columns.each_pair do |column, values| + column_name = "host|#{column}" + column_dimensions = { host: site.hostnames } + column_dimensions[column] = values + + rollup(name: column_name, dimensions: column_dimensions, filter: uri_dimensions) + reduce_rollup(name: column_name, dimensions: column_dimensions) + end + + stat.touch run_again! unless once end private + # Generar un rollup de access logs + # + # @param :name [String] + # @param :beginning [Time] + # @param :dimensions [Hash] + # @param :filter [Hash] + # @return [nil] + def rollup(name:, dimensions:, interval: starting_interval, filter: nil) + AccessLog.where(**(filter || dimensions)) + .where('created_at >= ?', beginning_of_interval) + .completed_requests + .non_robots + .group(*dimensions.keys) + .rollup(name, interval: interval, update: true) + end + + # Generar rollups con el resto de la información + # + # @param :name [String] + # @param :dimensions [Hash] + # @param :filter [Hash] + # @return [nil] + def reduce_rollup(name:, dimensions:, filter: nil) + Stat::INTERVALS.reduce do |_previous, current| + rollup(name: name, dimensions: dimensions, filter: filter, interval: current) + + current + end + + nil + end + def stat_name STAT_NAME end + # Obtiene todas las ubicaciones de archivos + # # @return [String] # # TODO: Cambiar al mergear origin-referer - def destination - @destination ||= site.deploys.find_by(type: 'DeployLocal').destination - end - - # TODO: Cambiar al mergear origin-referer - # - # @return [Array] - def hostnames - @hostnames ||= site.deploys.map do |deploy| - case deploy - when DeployLocal - site.hostname - when DeployWww - deploy.fqdn - when DeployAlternativeDomain - deploy.hostname.dup.tap do |h| - h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}") - end - when DeployHiddenService - deploy.onion - end - end.compact + def destinations + @destinations ||= site.deploys.map(&:destination).compact.select do |d| + File.directory?(d) + end.map do |d| + File.realpath(d) + end.uniq end # Recolecta todas las URIs menos imágenes # + # TODO: Para los sitios con DeployLocalizedDomain estamos buscando + # URIs de más. + # # @return [Array] def uris - @uris ||= Dir.chdir destination do - (Dir.glob('**/*.html') + Dir.glob('public/**/*').reject do |p| - File.directory? p - end.reject do |p| - p = p.downcase + @uris ||= + destinations.map do |destination| + locales.map do |locale| + uri = "/#{locale}/".squeeze('/') + dir = File.join(destination, locale) - IMAGES.any? do |i| - p.end_with? i + next unless File.directory? dir + + files(dir).map do |f| + uri + f + end end - end).map do |uri| - "/#{uri}" - end + end.flatten(3).compact + end + + # @return [Array] + def locales + @locales ||= ['', site.locales.map(&:to_s)].flatten(1) + end + + # @param :dir [String] + # @return [Array] + def files(dir) + Dir.chdir(dir) do + pages = Dir.glob('**/*.html') + files = Dir.glob('public/**/*') + files = remove_directories files + files = remove_images files + + [pages, files].flatten(1) + end + end + + # @param :files [Array] + # @return [Array] + def remove_directories(files) + files.reject do |f| + File.directory? f + end + end + + def remove_images(files) + files.reject do |f| + IMAGES.include? File.extname(f).downcase end end end diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index 4fa588f5..1b661059 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -28,15 +28,17 @@ class DeployLocal < Deploy # Obtener el tamaño de todos los archivos y directorios (los # directorios son archivos :) def size - paths = [destination, File.join(destination, '**', '**')] + @size ||= begin + paths = [destination, File.join(destination, '**', '**')] - Dir.glob(paths).map do |file| - if File.symlink? file - 0 - else - File.size(file) - end - end.inject(:+) + Dir.glob(paths).map do |file| + if File.symlink? file + 0 + else + File.size(file) + end + end.inject(:+) + end end def destination diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb new file mode 100644 index 00000000..996f8cdd --- /dev/null +++ b/app/models/deploy_rsync.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +# Sincroniza sitios a servidores remotos usando Rsync. El servidor +# remoto tiene que tener rsync instalado. +class DeployRsync < Deploy + store :values, accessors: %i[destination host_keys], coder: JSON + + def deploy + ssh? && rsync + end + + # El espacio remoto es el mismo que el local + # + # @return [Integer] + def size + deploy_local.size + end + + # Devolver el destino o lanzar un error si no está configurado + def destination + values[:destination].tap do |d| + raise(ArgumentError, 'destination no está configurado') if d.blank? + end + end + + private + + # Verificar la conexión SSH implementando Trust On First Use + # + # TODO: Medir el tiempo que tarda en iniciarse la conexión + # + # @return [Boolean] + def ssh? + user, host = user_host + ssh_available = false + + Net::SSH.start(host, user, verify_host_key: tofu, timeout: 5) do |ssh| + if values[:host_keys].blank? + # Guardar las llaves que se encontraron en la primera conexión + values[:host_keys] = ssh.transport.host_keys.map do |host_key| + "#{host_key.ssh_type} #{host_key.fingerprint}" + end + + ssh_available = save + else + ssh_available = true + end + end + + ssh_available + rescue Exception => e + ExceptionNotifier.notify_exception(e, data: { site: site.id, hostname: host, user: user }) + + false + end + + def env + { + 'HOME' => home_dir, + 'PATH' => '/usr/bin', + 'LANG' => ENV['LANG'] + } + end + + # Confiar en la primera llave que encontremos, fallar si cambian + # + # @return [Symbol] + def tofu + values[:host_keys].present? ? :always : :accept_new + end + + # Devuelve el par user host + # + # @return [Array] + def user_host + destination.split(':', 2).first.split('@', 2).tap do |d| + next unless d.size == 1 + + d.insert(0, nil) + end + end + + # Sincroniza hacia el directorio remoto + # + # @return [Boolean] + def rsync + run %(rsync -aviH --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + end + + # El origen es el destino de la compilación + # + # @return [String] + def source + deploy_local.destination + end + + def deploy_local + @deploy_local ||= site.deploys.find_by(type: 'DeployLocal') + end +end diff --git a/app/models/site.rb b/app/models/site.rb index 638b3f47..fd318995 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -101,6 +101,26 @@ class Site < ApplicationRecord "https://#{hostname}#{slash ? '/' : ''}" end + # TODO: Cambiar al mergear origin-referer + # + # @return [Array] + def hostnames + @hostnames ||= deploys.map do |deploy| + case deploy + when DeployLocal + hostname + when DeployWww + deploy.fqdn + when DeployAlternativeDomain + deploy.hostname.dup.tap do |h| + h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}") + end + when DeployHiddenService + deploy.onion + end + end.compact + end + # Obtiene los dominios alternativos # # @return Array @@ -123,7 +143,9 @@ class Site < ApplicationRecord # # @return Array def urls(slash: true) - alternative_urls(slash: slash) << url(slash: slash) + @urls ||= hostnames.map do |h| + "https://#{h}#{slash ? '/' : ''}" + end end def invitade?(usuarie) diff --git a/app/models/site_blazer.rb b/app/models/site_blazer.rb new file mode 100644 index 00000000..76dee12a --- /dev/null +++ b/app/models/site_blazer.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +SiteBlazer = Struct.new(:site) diff --git a/app/models/stat.rb b/app/models/stat.rb index 5f72ccd0..6c681aa4 100644 --- a/app/models/stat.rb +++ b/app/models/stat.rb @@ -3,8 +3,16 @@ # Registran cuándo fue la última recolección de datos. class Stat < ApplicationRecord # XXX: Los intervalos van en orden de mayor especificidad a menor - INTERVALS = %i[day month year].freeze + INTERVALS = %i[day].freeze RESOURCES = %i[builds space_used build_time].freeze + COLUMNS = %i[http_referer geoip2_data_country_name].freeze belongs_to :site + + # El intervalo por defecto + # + # @return [Symbol] + def self.default_interval + INTERVALS.first + end end diff --git a/app/models/usuarie.rb b/app/models/usuarie.rb index 6de7ba4b..c88dcc68 100644 --- a/app/models/usuarie.rb +++ b/app/models/usuarie.rb @@ -11,6 +11,8 @@ class Usuarie < ApplicationRecord has_many :roles has_many :sites, through: :roles + has_many :blazer_audits, foreign_key: 'user_id', class_name: 'Blazer::Audit' + has_many :blazer_queries, foreign_key: 'creator_id', class_name: 'Blazer::Query' def name email.split('@', 2).first diff --git a/app/policies/site_blazer_policy.rb b/app/policies/site_blazer_policy.rb new file mode 100644 index 00000000..a6ea01b7 --- /dev/null +++ b/app/policies/site_blazer_policy.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +# Les invitades no pueden ver las estadísticas (aun) +SiteBlazerPolicy = Struct.new(:usuarie, :site_blazer) do + def home? + site_blazer&.site&.usuarie? usuarie + end + + alias_method :show?, :home? +end diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 5e2fc706..22423bb8 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -9,6 +9,7 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do self.site = Site.new params add_role temporal: false, rol: 'usuarie' + sync_nodes I18n.with_locale(usuarie&.lang&.to_sym || I18n.default_locale) do site.save && @@ -144,4 +145,11 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do PostService.new(site: site, usuarie: usuarie, post: post, params: params).update end + + # Crea los deploys necesarios para sincronizar a otros nodos de Sutty + def sync_nodes + Rails.application.nodes.each do |node| + site.deploys.build(type: 'DeployRsync', destination: "sutty@#{node}:#{site.hostname}") + end + end end diff --git a/app/views/blazer/check_mailer/failing_checks.haml b/app/views/blazer/check_mailer/failing_checks.haml new file mode 100644 index 00000000..c28c3936 --- /dev/null +++ b/app/views/blazer/check_mailer/failing_checks.haml @@ -0,0 +1,5 @@ +%ul + - @checks.each do |check| + %li + = check.query.name + = check.state diff --git a/app/views/blazer/check_mailer/state_change.haml b/app/views/blazer/check_mailer/state_change.haml new file mode 100644 index 00000000..48418a58 --- /dev/null +++ b/app/views/blazer/check_mailer/state_change.haml @@ -0,0 +1,30 @@ +!!! +%html + %head + %meta{:content => "text/html; charset=UTF-8", "http-equiv" => "Content-Type"}/ + %body{:style => "font-family: 'Helvetica Neue', Arial, Helvetica; font-size: 14px; color: #333;"} + - if @error + %p= @error + - elsif @rows_count > 0 && @check_type == "bad_data" + %p + - if @rows_count <= 10 + = pluralize(@rows_count, "row") + - else + Showing 10 of #{@rows_count} rows + %table{:style => "width: 100%; border-spacing: 0; border-collapse: collapse;"} + %thead + %tr + - @columns.first(5).each do |column| + %th{:style => "padding: 8px; line-height: 1.4; text-align: left; vertical-align: bottom; border-bottom: 2px solid #ddd; width: #{(100 / @columns.size).round(2)}%;"} + = column + %tbody + - @rows.first(10).each do |row| + %tr + - @columns.first(5).each_with_index do |column, i| + %td{:style => "padding: 8px; line-height: 1.4; vertical-align: top; border-top: 1px solid #ddd;"} + - value = row[i] + - if @column_types[i] == "time" && value.to_s.length > 10 + - value = Time.parse(value).in_time_zone(Blazer.time_zone) rescue value + = value + - if @columns.size > 5 + %p{:style => "color: #999;"} Only first 5 columns shown diff --git a/app/views/blazer/queries/home.haml b/app/views/blazer/queries/home.haml new file mode 100644 index 00000000..977b6bda --- /dev/null +++ b/app/views/blazer/queries/home.haml @@ -0,0 +1,9 @@ +#queries + %table.table + %tbody.list + - @queries.each do |query| + %tr + -# + Por alguna razón no tenemos acceso a query_path para poder + generar la URL según Rails + %td= link_to query[:name], "/sites/#{params[:site_id]}/stats/queries/#{query.to_param}" diff --git a/app/views/blazer/queries/show.haml b/app/views/blazer/queries/show.haml new file mode 100644 index 00000000..3b5cb152 --- /dev/null +++ b/app/views/blazer/queries/show.haml @@ -0,0 +1,51 @@ +- blazer_title @query.name +.container + .row + .col-12 + %h1= @query.name + - if @query.description.present? + %p.lead= @query.description + - unless @result.chart_type.blank? + .col-12 + - case @result.chart_type + - when 'line' + = line_chart @chart_data, **@chart_options + - when 'line2' + = line_chart @chart_data, **@chart_options + - when 'pie' + = pie_chart @chart_data, **@chart_options + - when 'bar' + = column_chart @chart_data, **@chart_options + - when 'bar2' + = column_chart @chart_data, **@chart_options + - when 'scatter' + = scatter_chart @chart_data, **@chart_options + .col-12 + %table.table + %thead + %tr + - @result.columns.each do |key| + - next if key.include? 'ciphertext' + - next if key.include? 'encrypted' + %th.position-sticky.background-white{ style: 'top: 0' }= t("blazer.columns.#{key}", default: key.titleize) + %tbody + - @result.rows.each do |row| + %tr + - row.each_with_index do |v, i| + - k = @result.columns[i] + - next if k.include? 'ciphertext' + - next if k.include? 'encrypted' + %td + - if v.is_a?(Time) + - v = blazer_time_value(@data_source, k, v) + + - unless v.nil? + - if v.is_a?(String) && v.empty? + %span.text-muted= t('.empty') + - elsif @data_source.linked_columns[k] + = link_to blazer_format_value(k, v), @data_source.linked_columns[k].gsub('{value}', u(v.to_s)), target: '_blank' + - else + = blazer_format_value(k, v) + + - if (v2 = (@result.boom[k] || {})[v.nil? ? v : v.to_s]) + %span.text-muted= v2 diff --git a/app/views/deploys/_deploy_rsync.haml b/app/views/deploys/_deploy_rsync.haml new file mode 100644 index 00000000..0aab9802 --- /dev/null +++ b/app/views/deploys/_deploy_rsync.haml @@ -0,0 +1 @@ +-# nada diff --git a/app/views/layouts/_breadcrumb.haml b/app/views/layouts/_breadcrumb.haml index c4920bc7..dc0e3158 100644 --- a/app/views/layouts/_breadcrumb.haml +++ b/app/views/layouts/_breadcrumb.haml @@ -12,7 +12,7 @@ - else %span.line-clamp-1= link_to crumb.name, crumb.url - - if current_usuarie + - if @current_usuarie || current_usuarie %ul.navbar-nav - if @site&.tienda? %li.nav-item @@ -20,5 +20,5 @@ role: 'button', class: 'btn' %li.nav-item - = link_to t('.logout'), destroy_usuarie_session_path, + = link_to t('.logout'), main_app.destroy_usuarie_session_path, method: :delete, role: 'button', class: 'btn' diff --git a/app/views/layouts/blazer/application.haml b/app/views/layouts/blazer/application.haml new file mode 100644 index 00000000..add94190 --- /dev/null +++ b/app/views/layouts/blazer/application.haml @@ -0,0 +1,14 @@ +!!! +%html + %head + %meta{content: 'text/html; charset=UTF-8', 'http-equiv': 'Content-Type'}/ + %title= blazer_title ? blazer_title : 'Sutty' + %meta{charset: 'utf-8'}/ + = favicon_link_tag 'blazer/favicon.png' + = stylesheet_link_tag 'application' + = javascript_pack_tag 'blazer', 'data-turbolinks-track': 'reload' + = csrf_meta_tags + %body{ class: yield(:body) } + .container-fluid#sutty + = render 'layouts/breadcrumb' + = yield diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 90d65670..bc5c826c 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -15,6 +15,9 @@ - else %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'btn btn-secondary btn-sm' + - if policy(@site_stat).index? + = link_to t('stats.index.title'), site_stats_path(@site), class: 'btn' + - if policy(@site).edit? = link_to t('sites.edit.btn', site: @site.title), edit_site_path(@site), class: 'btn' @@ -71,8 +74,8 @@ %table.table{ data: { controller: 'reorder' } } %caption.sr-only= t('posts.caption') %thead - %tr - %th.border-0.background-white.position-sticky{ style: 'top: 0; z-index: 2', colspan: '4' } + %tr.sticky-top + %th.border-0{ colspan: '4' } .d-flex.flex-row.justify-content-between %div = submit_tag t('posts.reorder.submit'), class: 'btn' diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index bfcf33ef..49fbd023 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -6,32 +6,57 @@ %p %small = t('.last_update') - %time{ datetime: @last_stat.created_at } - #{time_ago_in_words @last_stat.created_at}. + %time{ datetime: @last_stat.updated_at } + #{time_ago_in_words @last_stat.updated_at}. - .mb-5 + %form.mb-5.form-inline{ method: 'get' } - Stat::INTERVALS.each do |interval| - = link_to t(".#{interval}"), site_stats_path(interval: interval, urls: params[:urls]), class: "btn #{'btn-primary active' if @interval == interval}" + = link_to t(".#{interval}"), site_stats_path(interval: interval, urls: params[:urls], period_start: params[:period_start].to_date.try(:"beginning_of_#{interval}").to_date, period_end: params[:period_end]), class: "mb-0 btn #{'btn-primary active' if @interval == interval}" + + %input.form-control{ type: 'date', name: :period_start, value: params[:period_start] } + %input.form-control{ type: 'date', name: :period_end, value: params[:period_end] } + %button.btn.mb-0{ type: 'submit' }= t('.filter') .mb-5 %h2= t('.host.title', count: @hostnames.size) %p.lead= t('.host.description') = line_chart site_stats_host_path(@chart_params), **@chart_options - .mb-5 + #custom-urls.mb-5 %h2= t('.urls.title') %p.lead= t('.urls.description') - %form + %form{ method: 'get', action: '#custom-urls' } %input{ type: 'hidden', name: 'interval', value: @interval } .form-group %label{ for: 'urls' }= t('.urls.label') - %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size, aria_describedby: 'help-urls' }= @normalized_urls.join("\n") + %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size + 1, aria_describedby: 'help-urls' }= @normalized_urls.join("\n") %small#help-urls.feedback.form-text.text-muted= t('.urls.help') .form-group %button.btn{ type: 'submit' }= t('.urls.submit') - if @normalized_urls.present? - = line_chart site_stats_uris_path(urls: params[:urls], **@chart_params), **@chart_options + = line_chart site_stats_uris_path(urls: @normalized_urls, **@chart_params), **@chart_options + .row.mb-5.row-cols-1.row-cols-md-2 + - @columns.each_pair do |column, values| + - next if values.blank? + .col.mb-5 + %h2= t(".columns.#{column}.title") + %p.lead= t(".columns.#{column}.description") + + %table.table + %colgroup + %col + %col + %thead + %tr.sticky-top + %th{ scope: 'col' }= t(".columns.#{column}.column") + %th{ scope: 'col' }= t('.columns.visits') + %tfoot + %tbody + - values.each_pair do |col, val| + %tr + %th{ scope: 'row', style: 'word-break: break-all' }= col.blank? ? t(".columns.#{column}.empty") : col + %td= val .mb-5 %h2= t('.resources.title') %p.lead= t('.resources.description') diff --git a/bin/access_logs b/bin/access_logs new file mode 100755 index 00000000..cfeeb57a --- /dev/null +++ b/bin/access_logs @@ -0,0 +1,15 @@ +#!/bin/sh +set -e + +# Volcar y eliminar todos los access logs de dos días atrás +date="`dateadd today -1d`" +file="/srv/http/_storage/${date}.psql.gz" +test -n "${date}" +test ! -f "${file}" + +psql -h postgresql "${DATABASE:-sutty}" sutty < "${file}" +begin; +copy (select * from access_logs where created_at < '${date}') to stdout; +delete from access_logs where created_at < '${date}'; +commit; +SQL diff --git a/config/application.rb b/config/application.rb index 031c1909..97ab244c 100644 --- a/config/application.rb +++ b/config/application.rb @@ -39,6 +39,7 @@ module Sutty config.active_storage.variant_processor = :vips config.to_prepare do + # Load application's model / class decorators Dir.glob(File.join(File.dirname(__FILE__), '..', 'app', '**', '*_decorator.rb')).sort.each do |c| Rails.configuration.cache_classes ? require(c) : load(c) end @@ -55,5 +56,9 @@ module Sutty EmailAddress::Config.error_messages translations.transform_keys(&:to_s), locale.to_s end end + + def nodes + @nodes ||= ENV.fetch('SUTTY_NODES', '').split(',') + end end end diff --git a/config/blazer.yml b/config/blazer.yml index 2ba0965f..11792ff1 100644 --- a/config/blazer.yml +++ b/config/blazer.yml @@ -50,7 +50,7 @@ user_method: current_usuarie user_name: email # custom before_action to use for auth -# before_action_method: require_admin +before_action_method: require_usuarie # email to send checks from from_email: blazer@<%= ENV.fetch('SUTTY', 'sutty.nl') %> diff --git a/config/locales/en.yml b/config/locales/en.yml index 530a9381..10a4793b 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -102,6 +102,10 @@ en: title: Alternative domain name success: Success! error: Error + deploy_rsync: + title: Synchronize to backup server + success: Success! + error: Error help: You can contact us by replying to this e-mail maintenance_mailer: notice: @@ -254,7 +258,7 @@ en: help: | These statistics show information about how your site is generated and how many resources it uses. - last_update: 'Updated every hour. Last update on ' + last_update: 'Updated daily. Last update on ' empty: 'There is no enough information yet. We invite you to come back in %{please_return_at}!' loading: 'Loading...' hour: 'Hourly' @@ -285,7 +289,19 @@ en: description: 'Average storage space used by your site.' build_time: title: 'Publication time' - description: 'Average time your site takes to build.' + description: 'Average time your site takes to build, from pressing "Publish changes" to actually being available on your site.' + columns: + visits: "Visits" + http_referer: + title: "Referers" + description: "Visits by origin" + column: "Referer" + empty: "(direct visit)" + geoip2_data_country_name: + title: "Countries" + description: "Visits by country" + column: "Country" + empty: "(couldn't detect country)" sites: donations: url: 'https://donaciones.sutty.nl/en/' @@ -612,3 +628,14 @@ en: edit: 'Editing' usuaries: index: 'Users' + stats: + index: 'Statistics' + blazer: + columns: + total: 'Total' + dia: 'Date' + date: 'Date' + visitas: 'Visits' + queries: + show: + empty: '(empty)' diff --git a/config/locales/es.yml b/config/locales/es.yml index eaa23137..02973de5 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -102,6 +102,10 @@ es: title: Dominio alternativo success: ¡Éxito! error: Hubo un error + deploy_rsync: + title: Sincronizar al servidor alternativo + success: ¡Éxito! + error: Hubo un error help: Por cualquier duda, responde este correo para contactarte con nosotres. maintenance_mailer: notice: @@ -259,7 +263,7 @@ es: help: | Las estadísticas visibilizan información sobre cómo se genera y cuántos recursos utiliza tu sitio. - last_update: 'Actualizadas cada hora. Última actualización hace ' + last_update: 'Actualizadas diariamente. Última actualización hace ' empty: 'Todavía no hay información suficiente. Te invitamos a volver en %{please_return_at} :)' loading: 'Cargando...' hour: 'Por hora' @@ -290,7 +294,19 @@ es: description: 'Espacio en disco que ocupa en promedio tu sitio.' build_time: title: 'Tiempo de publicación' - description: 'Tiempo promedio que toma en publicarse tu sitio.' + description: 'Tiempo que tarda el sitio en generarse, desde que usas el botón "Publicar cambios" hasta que los puedes ver en el sitio' + columns: + visits: "Visitas" + http_referer: + title: "Referencias" + description: "Orígenes de las visitas" + column: "Referencia" + empty: "(visita directa)" + geoip2_data_country_name: + title: "Países" + description: "Cantidad de visitas por país" + column: "País" + empty: "(no se pudo detectar el país)" sites: donations: url: 'https://donaciones.sutty.nl/' @@ -620,3 +636,14 @@ es: edit: 'Editando' usuaries: index: 'Usuaries' + stats: + index: 'Estadísticas' + blazer: + columns: + total: 'Total' + dia: 'Fecha' + date: 'Fecha' + visitas: 'Visitas' + queries: + show: + empty: '(vacío)' diff --git a/config/routes.rb b/config/routes.rb index 2aa0056f..8bab18af 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -4,8 +4,6 @@ Rails.application.routes.draw do devise_for :usuaries get '/.well-known/change-password', to: redirect('/usuaries/edit') - mount Blazer::Engine, at: 'blazer' - root 'application#index' constraints(Constraints::ApiSubdomain.new) do diff --git a/db/migrate/20200206163257_install_blazer.rb b/db/migrate/20200206163257_install_blazer.rb index 9b084169..32799053 100644 --- a/db/migrate/20200206163257_install_blazer.rb +++ b/db/migrate/20200206163257_install_blazer.rb @@ -3,8 +3,6 @@ # Blazer class InstallBlazer < ActiveRecord::Migration[6.0] def change - return unless Rails.env.production? - create_table :blazer_queries do |t| t.references :creator t.string :name diff --git a/db/migrate/20211022224008_add_site_to_stats.rb b/db/migrate/20211022224008_add_site_to_stats.rb new file mode 100644 index 00000000..db2b43ab --- /dev/null +++ b/db/migrate/20211022224008_add_site_to_stats.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +# La recolección de estadísticas podría pertenecer a un sitio +class AddSiteToStats < ActiveRecord::Migration[6.1] + def change + add_belongs_to :stats, :site, index: true, null: true + end +end diff --git a/db/migrate/20211022225449_add_name_to_stats.rb b/db/migrate/20211022225449_add_name_to_stats.rb new file mode 100644 index 00000000..89a17ee0 --- /dev/null +++ b/db/migrate/20211022225449_add_name_to_stats.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +# Agregarle un nombre a la estadística +class AddNameToStats < ActiveRecord::Migration[6.1] + def change + add_column :stats, :name, :string, null: false + add_index :stats, :name, using: 'hash' + end +end diff --git a/db/migrate/20220406211042_add_deploy_rsync_to_sites.rb b/db/migrate/20220406211042_add_deploy_rsync_to_sites.rb new file mode 100644 index 00000000..92b6f17b --- /dev/null +++ b/db/migrate/20220406211042_add_deploy_rsync_to_sites.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# Agrega un DeployRsync hacia los servidores alternativos para cada +# sitio +class AddDeployRsyncToSites < ActiveRecord::Migration[6.1] + def up + Site.find_each do |site| + SiteService.new(site: site).send :sync_nodes + site.save + end + end + + def down + DeployRsync.destroy_all + end +end diff --git a/db/schema.rb b/db/schema.rb index 107e7be7..a395329d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_05_14_165639) do +ActiveRecord::Schema.define(version: 2021_10_22_225449) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -64,6 +64,7 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do t.string "remote_user" t.boolean "crawler", default: false t.string "http_referer" + t.datetime "created_at", precision: 6 t.index ["geoip2_data_city_name"], name: "index_access_logs_on_geoip2_data_city_name" t.index ["geoip2_data_country_name"], name: "index_access_logs_on_geoip2_data_country_name" t.index ["host"], name: "index_access_logs_on_host" @@ -303,6 +304,15 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do t.index ["usuarie_id"], name: "index_roles_on_usuarie_id" end + create_table "rollups", force: :cascade do |t| + t.string "name", null: false + t.string "interval", null: false + t.datetime "time", null: false + t.jsonb "dimensions", default: {}, null: false + t.float "value" + t.index ["name", "interval", "time", "dimensions"], name: "index_rollups_on_name_and_interval_and_time_and_dimensions", unique: true + end + create_table "sites", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -324,6 +334,15 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do t.index ["name"], name: "index_sites_on_name", unique: true end + create_table "stats", force: :cascade do |t| + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.bigint "site_id" + t.string "name", null: false + t.index ["name"], name: "index_stats_on_name", using: :hash + t.index ["site_id"], name: "index_stats_on_site_id" + end + create_table "usuaries", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -370,4 +389,10 @@ new.indexed_content := to_tsvector(('pg_catalog.' || new.dictionary)::regconfig, SQL_ACTIONS end + create_trigger("access_logs_before_insert_row_tr", :compatibility => 1). + on("access_logs"). + before(:insert) do + "new.created_at := to_timestamp(new.msec);" + end + end diff --git a/lib/tasks/stats.rake b/lib/tasks/stats.rake new file mode 100644 index 00000000..9461782a --- /dev/null +++ b/lib/tasks/stats.rake @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +namespace :stats do + desc 'Process stats' + task process_all: :environment do + Site.all.pluck(:id).each do |site_id| + UriCollectionJob.perform_now site_id: site_id, once: true + StatCollectionJob.perform_now site_id: site_id, once: true + end + end +end diff --git a/monit.conf b/monit.conf index 96c08d8a..83d17449 100644 --- a/monit.conf +++ b/monit.conf @@ -25,3 +25,13 @@ check program blazer with path "/usr/local/bin/sutty blazer" every 61 cycles if status != 0 then alert + +check program access_logs + with path "/srv/http/bin/access_logs" as uid "app" and gid "www-data" + every "0 0 * * *" + if status != 0 then alert + +check program stats + with path "/usr/bin/foreman run -f /srv/Procfile -d /srv stats" as uid "rails" gid "www-data" + every "0 1 * * *" + if status != 0 then alert diff --git a/yarn.lock b/yarn.lock index ee021266..75d0138a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2144,7 +2144,7 @@ collection-visit@^1.0.0: map-visit "^1.0.0" object-visit "^1.0.0" -color-convert@^1.9.0, color-convert@^1.9.3: +color-convert@^1.9.0, color-convert@^1.9.1, color-convert@^1.9.3: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== @@ -4692,6 +4692,11 @@ mkdirp@^1.0.3, mkdirp@^1.0.4: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== +moment@^2.10.2: + version "2.29.1" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3" + integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ== + move-concurrently@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92"