diff --git a/.profile b/.profile new file mode 100644 index 00000000..3c73ffa9 --- /dev/null +++ b/.profile @@ -0,0 +1,9 @@ +Color_Off='\e[0m' +BPurple='\e[1;35m' +BBlue='\e[1;34m' + +is_git() { + git rev-parse --abbrev-ref HEAD 2>/dev/null +} + +PS1="\[${BPurple}\]\$(is_git) \[${BBlue}\]\W\[${Color_Off}\] >_ " diff --git a/Gemfile b/Gemfile index 2b304ee0..1e476dde 100644 --- a/Gemfile +++ b/Gemfile @@ -64,7 +64,7 @@ gem 'rails-i18n' gem 'rails_warden' gem 'redis', require: %w[redis redis/connection/hiredis] gem 'redis-rails' -gem 'rollups', git: 'https://github.com/ankane/rollup.git', branch: 'master' +gem 'rollups', git: 'https://github.com/fauno/rollup.git', branch: 'update' gem 'rubyzip' gem 'rugged' gem 'concurrent-ruby-ext' diff --git a/Gemfile.lock b/Gemfile.lock index 8df2d77e..87812726 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,15 +6,6 @@ GIT rails (>= 3.0) rake (>= 0.8.7) -GIT - remote: https://github.com/ankane/rollup.git - revision: 0ab6c603450175eb1004f7793e86486943cb9f72 - branch: master - specs: - rollups (0.1.3) - activesupport (>= 5.1) - groupdate (>= 5.2) - GIT remote: https://github.com/fauno/email_address revision: 536b51f7071b68a55140c0c1726b4cd401d1c04d @@ -24,6 +15,15 @@ GIT netaddr (>= 2.0.4, < 3) simpleidn +GIT + remote: https://github.com/fauno/rollup.git + revision: ddbb345aa57e63b4cfdf7557267efa89ba60caac + branch: update + specs: + rollups (0.1.3) + activesupport (>= 5.1) + groupdate (>= 5.2) + GEM remote: https://gems.sutty.nl/ specs: @@ -214,8 +214,8 @@ GEM ffi (~> 1.0) globalid (0.6.0) activesupport (>= 5.0) - groupdate (5.2.2) - activesupport (>= 5) + groupdate (6.1.0) + activesupport (>= 5.2) hairtrigger (0.2.24) activerecord (>= 5.0, < 7) ruby2ruby (~> 2.4) diff --git a/Procfile b/Procfile index b308ffd5..8f6c7741 100644 --- a/Procfile +++ b/Procfile @@ -5,3 +5,4 @@ blazer_1h: bundle exec rake blazer:run_checks SCHEDULE="1 hour" blazer_1d: bundle exec rake blazer:run_checks SCHEDULE="1 day" blazer: bundle exec rake blazer:send_failing_checks prometheus: bundle exec prometheus_exporter -b 0.0.0.0 --prefix "sutty_" +stats: bundle exec rake stats:process_all diff --git a/app/assets/stylesheets/application.scss b/app/assets/stylesheets/application.scss index e806a032..b756759a 100644 --- a/app/assets/stylesheets/application.scss +++ b/app/assets/stylesheets/application.scss @@ -126,6 +126,7 @@ ol.breadcrumb { color: var(--foreground); } +.table tr.sticky-top, .form-control, .custom-file-label { background-color: var(--background); diff --git a/app/controllers/active_storage/direct_uploads_controller_decorator.rb b/app/controllers/active_storage/direct_uploads_controller_decorator.rb new file mode 100644 index 00000000..c62dae2a --- /dev/null +++ b/app/controllers/active_storage/direct_uploads_controller_decorator.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module ActiveStorage + # Modifica la creación de un blob antes de subir el archivo para que + # incluya el JekyllService adecuado. + module DirectUploadsControllerDecorator + extend ActiveSupport::Concern + + included do + def create + blob = ActiveStorage::Blob.create_before_direct_upload!(service_name: session[:service_name], **blob_args) + render json: direct_upload_json(blob) + end + + private + + # Normalizar los caracteres unicode en los nombres de archivos + # para que puedan propagarse correctamente a través de todo el + # stack. + def blob_args + params.require(:blob).permit(:filename, :byte_size, :checksum, :content_type, metadata: {}).to_h.symbolize_keys.tap do |ba| + ba[:filename] = ba[:filename].unicode_normalize + end + end + end + end +end + +ActiveStorage::DirectUploadsController.include ActiveStorage::DirectUploadsControllerDecorator diff --git a/app/controllers/active_storage/disk_controller_decorator.rb b/app/controllers/active_storage/disk_controller_decorator.rb new file mode 100644 index 00000000..14366a15 --- /dev/null +++ b/app/controllers/active_storage/disk_controller_decorator.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module ActiveStorage + # Modificar {DiskController} para poder asociar el blob a un sitio + module DiskControllerDecorator + extend ActiveSupport::Concern + + included do + # Asociar el archivo subido al sitio correspondiente. Cada sitio + # tiene su propio servicio de subida de archivos. + def update + if (token = decode_verified_token) + if acceptable_content?(token) + named_disk_service(token[:service_name]).upload token[:key], request.body, checksum: token[:checksum] + + blob = ActiveStorage::Blob.find_by_key token[:key] + site = Site.find_by_name token[:service_name] + + site.static_files.attach(blob) + else + head :unprocessable_entity + end + else + head :not_found + end + rescue ActiveStorage::IntegrityError + head :unprocessable_entity + end + end + end +end + +ActiveStorage::DiskController.include ActiveStorage::DiskControllerDecorator diff --git a/app/controllers/api/v1/csp_reports_controller.rb b/app/controllers/api/v1/csp_reports_controller.rb index bc6cfae0..ea186729 100644 --- a/app/controllers/api/v1/csp_reports_controller.rb +++ b/app/controllers/api/v1/csp_reports_controller.rb @@ -6,19 +6,22 @@ module Api class CspReportsController < BaseController skip_forgery_protection + # No queremos indicar que algo salió mal + rescue_from ActionController::ParameterMissing, with: :csp_report_created + # Crea un reporte de CSP intercambiando los guiones medios por # bajos # # TODO: Aplicar rate_limit def create - csp = CspReport.new(csp_report_params.to_h.map do |k, v| - [k.tr('-', '_'), v] - end.to_h) + csp = CspReport.new(csp_report_params.to_h.transform_keys do |k| + k.tr('-', '_') + end) csp.id = SecureRandom.uuid csp.save - render json: {}, status: :created + csp_report_created end private @@ -39,6 +42,10 @@ module Api :'column-number', :'source-file') end + + def csp_report_created + render json: {}, status: :created + end end end end diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index acd0134d..d8498218 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -3,6 +3,7 @@ # Forma de ingreso a Sutty class ApplicationController < ActionController::Base include ExceptionHandler + include Pundit protect_from_forgery with: :null_session, prepend: true @@ -10,6 +11,7 @@ class ApplicationController < ActionController::Base before_action :configure_permitted_parameters, if: :devise_controller? around_action :set_locale + rescue_from Pundit::NilPolicyError, with: :page_not_found rescue_from ActionController::RoutingError, with: :page_not_found rescue_from ActionController::ParameterMissing, with: :page_not_found @@ -33,7 +35,7 @@ class ApplicationController < ActionController::Base def find_site id = params[:site_id] || params[:id] - unless (site = current_usuarie.sites.find_by_name(id)) + unless (site = current_usuarie&.sites&.find_by_name(id)) raise SiteNotFound end @@ -62,6 +64,21 @@ class ApplicationController < ActionController::Base render 'application/page_not_found', status: :not_found end + # Necesario para poder acceder a Blazer. Solo les usuaries de este + # sitio pueden acceder al panel. + def require_usuarie + site = find_site + authorize SiteBlazer.new(site) + + # Necesario para los breadcrumbs. + ActionView::Base.include Loaf::ViewExtensions unless ActionView::Base.included_modules.include? Loaf::ViewExtensions + + breadcrumb current_usuarie.email, main_app.edit_usuarie_registration_path + breadcrumb 'sites.index', main_app.sites_path, match: :exact + breadcrumb site.title, main_app.site_path(site), match: :exact + breadcrumb 'stats.index', root_path, match: :exact + end + protected def configure_permitted_parameters diff --git a/app/controllers/concerns/blazer_decorator.rb b/app/controllers/concerns/blazer_decorator.rb new file mode 100644 index 00000000..876f423d --- /dev/null +++ b/app/controllers/concerns/blazer_decorator.rb @@ -0,0 +1,194 @@ +# frozen_string_literal: true + +# Modificaciones para Blazer +module BlazerDecorator + # No poder obtener información de la base de datos. + module DisableDatabaseInfo + extend ActiveSupport::Concern + + included do + def docs; end + + def tables; end + + def schema; end + end + end + + # Deshabilitar edición de consultas y chequeos. + module DisableEdits + extend ActiveSupport::Concern + + included do + def create; end + + def update; end + + def destroy; end + + def run; end + + def refresh; end + + def cancel; end + end + end + + # Blazer hace un gran esfuerzo para ejecutar consultas de forma + # asincrónica pero termina enviándolas por JS. + module RunSync + extend ActiveSupport::Concern + + included do + alias_method :original_show, :show + + include Blazer::BaseHelper + + def show + original_show + + options = { user: blazer_user, query: @query, run_id: SecureRandom.uuid, async: false } + @data_source = Blazer.data_sources[@query.data_source] + @result = Blazer::RunStatement.new.perform(@data_source, @statement, options) + chart_data + end + + private + + # Solo mostrar las consultas de le usuarie + def set_queries(_ = nil) + @queries = (@current_usuarie || current_usuarie).blazer_queries + end + + # blazer-2.4.2/app/views/blazer/queries/run.html.erb + def chart_type + case @result.chart_type + when /\Aline(2)?\z/ + chart_options.merge! min: nil + when /\Abar(2)?\z/ + chart_options.merge! library: { tooltips: { intersect: false, axis: 'x' } } + when 'pie' + chart_options + when 'scatter' + chart_options.merge! library: { tooltips: { intersect: false } }, xtitle: @result.columns[0], + ytitle: @result.columns[1] + when nil + else + if @result.column_types.size == 2 + chart_options.merge! library: { tooltips: { intersect: false, axis: 'x' } } + else + chart_options.merge! library: { tooltips: { intersect: false } } + end + end + + @result.chart_type + end + + def chart_data + @chart_data ||= + case chart_type + when 'line' + @result.columns[1..-1].each_with_index.map do |k, i| + { + name: blazer_series_name(k), + data: @result.rows.map do |r| + [r[0], r[i + 1]] + end, + library: series_library[i] + } + end + when 'line2' + @result.rows.group_by do |r| + v = r[1] + (@result.boom[@result.columns[1]] || {})[v.to_s] || v + end.each_with_index.map do |(name, v), i| + { + name: blazer_series_name(name), + data: v.map do |v2| + [v2[0], v2[2]] + end, + library: series_library[i] + } + end + when 'pie' + @result.rows.map do |r| + [(@result.boom[@result.columns[0]] || {})[r[0].to_s] || r[0], r[1]] + end + when 'bar' + (@result.rows.first.size - 1).times.map do |i| + name = @result.columns[i + 1] + + { + name: blazer_series_name(name), + data: @result.rows.first(20).map do |r| + [(@result.boom[@result.columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]] + end + } + end + when 'bar2' + first_20 = @result.rows.group_by { |r| r[0] }.values.first(20).flatten(1) + labels = first_20.map { |r| r[0] }.uniq + series = first_20.map { |r| r[1] }.uniq + labels.each do |l| + series.each do |s| + first_20 << [l, s, 0] unless first_20.find { |r| r[0] == l && r[1] == s } + end + end + + first_20.group_by do |r| + v = r[1] + (@result.boom[@result.columns[1]] || {})[v.to_s] || v + end.each_with_index.map do |(name, v), _i| + { + name: blazer_series_name(name), + data: v.sort_by do |r2| + labels.index(r2[0]) + end.map do |v2| + v3 = v2[0] + [(@result.boom[@result.columns[0]] || {})[v3.to_s] || v3, v2[2]] + end + } + end + when 'scatter' + @result.rows + end + end + + def target_index + @target_index ||= @result.columns.index do |k| + k.downcase == 'target' + end + end + + def series_library + @series_library ||= {}.tap do |sl| + if target_index + color = '#109618' + sl[target_index - 1] = { + pointStyle: 'line', + hitRadius: 5, + borderColor: color, + pointBackgroundColor: color, + backgroundColor: color, + pointHoverBackgroundColor: color + } + end + end + end + + def chart_options + @chart_options ||= { id: SecureRandom.hex } + end + end + end +end + +classes = [Blazer::QueriesController, Blazer::ChecksController, Blazer::DashboardsController] +modules = [BlazerDecorator::DisableDatabaseInfo, BlazerDecorator::DisableEdits] +classes.each do |klass| + modules.each do |modul| + klass.include modul unless klass.included_modules.include? modul + end +end + +Blazer::QueriesController.include BlazerDecorator::RunSync diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index dbdd4d0a..c5dc0f54 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -2,10 +2,8 @@ # Controlador para artículos class PostsController < ApplicationController - include Pundit - rescue_from Pundit::NilPolicyError, with: :page_not_found - before_action :authenticate_usuarie! + before_action :service_for_direct_upload, only: %i[new edit] # TODO: Traer los comunes desde ApplicationController breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path @@ -37,6 +35,8 @@ class PostsController < ApplicationController # Filtrar los posts que les invitades no pueden ver @usuarie = site.usuarie? current_usuarie + + @site_stat = SiteStat.new(site) end def show @@ -166,4 +166,9 @@ class PostsController < ApplicationController def post @post ||= site.posts(lang: locale).find(params[:post_id] || params[:id]) end + + # Recuerda el nombre del servicio de subida de archivos + def service_for_direct_upload + session[:service_name] = site.name.to_sym + end end diff --git a/app/controllers/private_controller.rb b/app/controllers/private_controller.rb index bb4d782d..01b6888c 100644 --- a/app/controllers/private_controller.rb +++ b/app/controllers/private_controller.rb @@ -6,8 +6,6 @@ class PrivateController < ApplicationController # XXX: Permite ejecutar JS skip_forgery_protection - include Pundit - # Enviar el archivo si existe, agregar una / al final siempre para no # romper las direcciones relativas. def show diff --git a/app/controllers/sites_controller.rb b/app/controllers/sites_controller.rb index bdaa9011..b4826226 100644 --- a/app/controllers/sites_controller.rb +++ b/app/controllers/sites_controller.rb @@ -2,9 +2,6 @@ # Controlador de sitios class SitesController < ApplicationController - include Pundit - rescue_from Pundit::NilPolicyError, with: :page_not_found - before_action :authenticate_usuarie! breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path diff --git a/app/controllers/stats_controller.rb b/app/controllers/stats_controller.rb index 44073c1f..c2c7bc58 100644 --- a/app/controllers/stats_controller.rb +++ b/app/controllers/stats_controller.rb @@ -8,6 +8,10 @@ class StatsController < ApplicationController before_action :authenticate_usuarie! before_action :authorize_stats + breadcrumb -> { current_usuarie.email }, :edit_usuarie_registration_path + breadcrumb 'sites.index', :sites_path, match: :exact + breadcrumb -> { site.title }, -> { site_posts_path(site, locale: locale) }, match: :exact + EXTRA_OPTIONS = { builds: {}, space_used: { bytes: true }, @@ -20,19 +24,53 @@ class StatsController < ApplicationController policy.script_src :self, :unsafe_inline end + # Parámetros por defecto + # + # @return [Hash] + def default_url_options + { interval: 'day', period_start: Date.today.beginning_of_year, period_end: Date.today } + end + def index - @chart_params = { interval: interval } + breadcrumb I18n.t('stats.index.title'), '' + + params.with_defaults! default_url_options + + @chart_params = { + interval: interval, + period_start: params[:period_start], + period_end: params[:period_end] + } + hostnames last_stat chart_options normalized_urls + + expires_in = Time.now.try(:"end_of_#{Stat.default_interval}") - Time.now + @columns = {} + + Stat::COLUMNS.each do |column| + @columns[column] = + Rails.cache.fetch("stats/#{column}/#{site.id}", expires_in: expires_in) do + rollup_scope.where(interval: interval, name: "host|#{column}") + .where_dimensions(host: hostnames) + .group("dimensions->>'#{column}'") + .order('sum(value) desc') + .sum(:value) + .transform_values(&:to_i) + .transform_values do |v| + v * nodes + end + end + end end # Genera un gráfico de visitas por dominio asociado a este sitio def host - return unless stale? [last_stat, hostnames, interval] + return unless stale? [last_stat, hostnames, interval, period] - stats = Rollup.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series| + stats = rollup_scope.where_dimensions(host: hostnames).multi_series('host', interval: interval).tap do |series| series.each do |serie| serie[:name] = serie.dig(:dimensions, 'host') serie[:data].transform_values! do |value| @@ -45,23 +83,20 @@ class StatsController < ApplicationController end def resources - return unless stale? [last_stat, interval, resource] + return unless stale? [last_stat, interval, resource, period] - options = { - interval: interval, - dimensions: { - deploy_id: @site.deploys.where(type: 'DeployLocal').pluck(:id).first - } - } + options = { interval: interval, dimensions: { site_id: site.id } } - render json: Rollup.series(resource, **options) + render json: rollup_scope.series(resource, **options) end def uris - return unless stale? [last_stat, hostnames, interval, normalized_urls] + return unless stale? [last_stat, hostnames, interval, normalized_urls, period] options = { host: hostnames, uri: normalized_paths } - stats = Rollup.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series| + # XXX: where_dimensions es más corto pero no aprovecha los índices + # de Rollup + stats = rollup_scope.where_dimensions(**options).multi_series('host|uri', interval: interval).tap do |series| series.each do |serie| serie[:name] = serie[:dimensions].slice('host', 'uri').values.join.sub('/index.html', '/') serie[:data].transform_values! do |value| @@ -75,34 +110,44 @@ class StatsController < ApplicationController private + def rollup_scope + Rollup.where(time: period) + end + def last_stat - @last_stat ||= Stat.last + @last_stat ||= site.stats.last end def authorize_stats - @site = find_site - authorize SiteStat.new(@site) + authorize SiteStat.new(site) end # TODO: Eliminar cuando mergeemos referer-origin def hostnames - @hostnames ||= [@site.hostname, @site.alternative_hostnames].flatten + @hostnames ||= site.hostnames end # Normalizar las URLs # # @return [Array] def normalized_urls - @normalized_urls ||= params.permit(:urls).try(:[], - :urls)&.split("\n")&.map(&:strip)&.select(&:present?)&.select do |uri| - uri.start_with? 'https://' - end&.map do |u| - # XXX: Eliminar - # @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1} - next u unless u.end_with? '/' + @normalized_urls ||= + begin + urls = params[:urls].is_a?(Array) ? params[:urls] : params[:urls]&.split("\n") + urls = urls&.map(&:strip)&.select(&:present?)&.select do |uri| + uri.start_with? 'https://' + end - "#{u}index.html" - end&.uniq || [@site.url, @site.urls].flatten.uniq + urls ||= [site.url] + + urls.map do |u| + # XXX: Eliminar al deployear + # @see {https://0xacab.org/sutty/containers/nginx/-/merge_requests/1} + next u unless u.end_with? '/' + + "#{u}index.html" + end.uniq + end end def normalized_paths @@ -140,14 +185,15 @@ class StatsController < ApplicationController def interval @interval ||= begin i = params[:interval]&.to_sym - Stat::INTERVALS.include?(i) ? i : :day + Stat::INTERVALS.include?(i) ? i : Stat::INTERVALS.first end end + # @return [Symbol] def resource @resource ||= begin r = params[:resource].to_sym - Stat::RESOURCES.include?(r) ? r : :builds + Stat::RESOURCES.include?(r) ? r : Stat::RESOURCES.first end end @@ -165,4 +211,15 @@ class StatsController < ApplicationController def nodes @nodes ||= ENV.fetch('NODES', 1).to_i end + + def period + @period ||= begin + p = params.permit(:period_start, :period_end) + p[:period_start]..p[:period_end] + end + end + + def site + @site ||= find_site + end end diff --git a/app/javascript/editor/types/multimedia.ts b/app/javascript/editor/types/multimedia.ts index 2af9643a..29d2a0ab 100644 --- a/app/javascript/editor/types/multimedia.ts +++ b/app/javascript/editor/types/multimedia.ts @@ -137,8 +137,10 @@ export function setupAuxiliaryToolbar(editor: Editor): void { "click", (event) => { const files = editor.toolbar.auxiliary.multimedia.fileEl.files; - if (!files || !files.length) - throw new Error("no hay archivos para subir"); + if (!files || !files.length) { + console.info("no hay archivos para subir"); + return; + } const file = files[0]; const selectedEl = editor.contentEl.querySelector( diff --git a/app/javascript/editor/types/parentBlocks.ts b/app/javascript/editor/types/parentBlocks.ts index ffe40bdf..5ba8f366 100644 --- a/app/javascript/editor/types/parentBlocks.ts +++ b/app/javascript/editor/types/parentBlocks.ts @@ -20,7 +20,6 @@ function makeParentBlock( }; } -// TODO: añadir blockquote // XXX: si agregás algo acá, probablemente le quieras hacer un botón // en app/views/posts/attributes/_content.haml export const parentBlocks: { [propName: string]: EditorNode } = { diff --git a/app/jobs/concerns/recursive_rollup.rb b/app/jobs/concerns/recursive_rollup.rb new file mode 100644 index 00000000..1da14c01 --- /dev/null +++ b/app/jobs/concerns/recursive_rollup.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +# Implementa rollups recursivos +module RecursiveRollup + extend ActiveSupport::Concern + + included do + private + + # Genera un rollup recursivo en base al período anterior y aplica una + # operación. + # + # @param :name [String] + # @param :interval_previous [String] + # @param :interval [String] + # @param :operation [Symbol] + # @param :dimensions [Hash] + # @param :beginning [Time] + # @return [Rollup] + def recursive_rollup(name:, interval_previous:, interval:, dimensions:, beginning:, operation: :sum) + Rollup.where(name: name, interval: interval_previous, dimensions: dimensions) + .where('time >= ?', beginning.try(:"beginning_of_#{interval}")) + .group(*dimensions_to_jsonb_query(dimensions)) + .rollup(name, interval: interval, update: true) do |rollup| + rollup.try(operation, :value) + end + end + + # Reducir las estadísticas calculadas aplicando un rollup sobre el + # intervalo más amplio. + # + # @param :name [String] + # @param :operation [Symbol] + # @param :dimensions [Hash] + # @return [nil] + def reduce_rollup(name:, dimensions:, operation: :sum) + Stat::INTERVALS.reduce do |previous, current| + recursive_rollup(name: name, + interval_previous: previous, + interval: current, + dimensions: dimensions, + beginning: beginning_of_interval, + operation: operation) + + # Devolver el intervalo actual + current + end + + nil + end + + # @param :dimensions [Hash] + # @return [Array] + def dimensions_to_jsonb_query(dimensions) + dimensions.keys.map do |key| + "dimensions->'#{key}'" + end + end + end +end diff --git a/app/jobs/periodic_job.rb b/app/jobs/periodic_job.rb index 8d9453a3..2f60a2b3 100644 --- a/app/jobs/periodic_job.rb +++ b/app/jobs/periodic_job.rb @@ -52,4 +52,12 @@ class PeriodicJob < ApplicationJob def beginning_of_interval @beginning_of_interval ||= last_stat.created_at.try(:"beginning_of_#{starting_interval}") end + + def stop_file + @stop_file ||= Rails.root.join('tmp', self.class.to_s.tableize) + end + + def stop? + File.exist? stop_file + end end diff --git a/app/jobs/stat_collection_job.rb b/app/jobs/stat_collection_job.rb index 2aa8d702..e402e3b5 100644 --- a/app/jobs/stat_collection_job.rb +++ b/app/jobs/stat_collection_job.rb @@ -2,11 +2,15 @@ # Genera resúmenes de información para poder mostrar estadísticas y se # corre regularmente a sí misma. -class StatCollectionJob < ApplicationJob +class StatCollectionJob < PeriodicJob + include RecursiveRollup + STAT_NAME = 'stat_collection_job' def perform(site_id:, once: true) @site = Site.find site_id + beginning = beginning_of_interval + stat = site.stats.create! name: STAT_NAME scope.rollup('builds', **options) @@ -18,44 +22,23 @@ class StatCollectionJob < ApplicationJob rollup.average(:seconds) end - # XXX: Es correcto promediar promedios? - Stat::INTERVALS.reduce do |previous, current| - rollup(name: 'builds', interval_previous: previous, interval: current) - rollup(name: 'space_used', interval_previous: previous, interval: current, operation: :average) - rollup(name: 'build_time', interval_previous: previous, interval: current, operation: :average) + dimensions = { site_id: site_id } - current - end - - # Registrar que se hicieron todas las recolecciones - site.stats.create! name: STAT_NAME + reduce_rollup(name: 'builds', operation: :sum, dimensions: dimensions) + reduce_rollup(name: 'space_used', operation: :average, dimensions: dimensions) + reduce_rollup(name: 'build_time', operation: :average, dimensions: dimensions) + stat.touch run_again! unless once end private - # Genera un rollup recursivo en base al período anterior y aplica una - # operación. - # - # @return [NilClass] - def rollup(name:, interval_previous:, interval:, operation: :sum) - Rollup.where(name: name, interval: interval_previous) - .where_dimensions(site_id: site.id) - .group("dimensions->'site_id'") - .rollup(name, interval: interval, update: true) do |rollup| - rollup.try(:operation, :value) - end - end - # Los registros a procesar # # @return [ActiveRecord::Relation] def scope - @scope ||= site.build_stats - .jekyll - .where('created_at => ?', beginning_of_interval) - .group(:site_id) + @scope ||= site.build_stats.jekyll.where('build_stats.created_at >= ?', beginning_of_interval).group(:site_id) end # Las opciones por defecto @@ -64,4 +47,8 @@ class StatCollectionJob < ApplicationJob def options @options ||= { interval: starting_interval, update: true } end + + def stat_name + STAT_NAME + end end diff --git a/app/jobs/uri_collection_job.rb b/app/jobs/uri_collection_job.rb index 9ec333cd..4cbbf593 100644 --- a/app/jobs/uri_collection_job.rb +++ b/app/jobs/uri_collection_job.rb @@ -13,94 +13,160 @@ class UriCollectionJob < PeriodicJob # Ignoramos imágenes porque suelen ser demasiadas y no aportan a las # estadísticas. - IMAGES = %w[.png .jpg .jpeg .gif .webp].freeze + IMAGES = %w[.png .jpg .jpeg .gif .webp .jfif].freeze STAT_NAME = 'uri_collection_job' def perform(site_id:, once: true) @site = Site.find site_id - hostnames.each do |hostname| - uris.each do |uri| - return if File.exist? Rails.root.join('tmp', 'uri_collection_job_stop') + # Obtener el principio del intervalo anterior + beginning_of_interval + # Recordar la última vez que se corrió la tarea + stat = site.stats.create! name: STAT_NAME + # Columnas a agrupar + columns = Stat::COLUMNS.zip([nil]).to_h - AccessLog.where(host: hostname, uri: uri) - .where('created_at >= ?', beginning_of_interval) - .completed_requests - .non_robots - .group(:host, :uri) - .rollup('host|uri', interval: starting_interval, update: true) + # Las URIs son la fuente de verdad de las visitas, porque son las + # que indican las páginas y recursos descargables, el resto son + # imágenes, CSS, JS y tipografías que no nos aportan números + # significativos. + uri_dimensions = { host: site.hostnames, uri: uris } + host_dimensions = { host: site.hostnames } - # Reducir las estadísticas calculadas aplicando un rollup sobre el - # intervalo más amplio. - Stat::INTERVALS.reduce do |previous, current| - Rollup.where(name: 'host|uri', interval: previous) - .where_dimensions(host: hostname, uri: uri) - .group("dimensions->'host'", "dimensions->'uri'") - .rollup('host|uri', interval: current, update: true) do |rollup| - rollup.sum(:value) - end - - # Devolver el intervalo actual - current - end - end + # Recorremos todos los hostnames y uris posibles y luego agrupamos + # recursivamente para no tener que recalcular, asumiendo que es más + # rápido buscar en los rollups indexados que en la tabla en bruto. + # + # Los referers solo se agrupan por host. + columns.each_key do |column| + columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) end - # Recordar la última vez que se corrió la tarea - site.stats.create! name: STAT_NAME + # Cantidad de visitas por host + rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) + reduce_rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions) + + # Cantidad de visitas por página/recurso + rollup(name: 'host|uri', dimensions: uri_dimensions) + reduce_rollup(name: 'host|uri', dimensions: uri_dimensions) + + # Cantidad de visitas host y parámetro + columns.each_pair do |column, values| + column_name = "host|#{column}" + column_dimensions = { host: site.hostnames } + column_dimensions[column] = values + + rollup(name: column_name, dimensions: column_dimensions, filter: uri_dimensions) + reduce_rollup(name: column_name, dimensions: column_dimensions) + end + + stat.touch run_again! unless once end private + # Generar un rollup de access logs + # + # @param :name [String] + # @param :beginning [Time] + # @param :dimensions [Hash] + # @param :filter [Hash] + # @return [nil] + def rollup(name:, dimensions:, interval: starting_interval, filter: nil) + AccessLog.where(**(filter || dimensions)) + .where('created_at >= ?', beginning_of_interval) + .completed_requests + .non_robots + .group(*dimensions.keys) + .rollup(name, interval: interval, update: true) + end + + # Generar rollups con el resto de la información + # + # @param :name [String] + # @param :dimensions [Hash] + # @param :filter [Hash] + # @return [nil] + def reduce_rollup(name:, dimensions:, filter: nil) + Stat::INTERVALS.reduce do |_previous, current| + rollup(name: name, dimensions: dimensions, filter: filter, interval: current) + + current + end + + nil + end + def stat_name STAT_NAME end + # Obtiene todas las ubicaciones de archivos + # # @return [String] # # TODO: Cambiar al mergear origin-referer - def destination - @destination ||= site.deploys.find_by(type: 'DeployLocal').destination - end - - # TODO: Cambiar al mergear origin-referer - # - # @return [Array] - def hostnames - @hostnames ||= site.deploys.map do |deploy| - case deploy - when DeployLocal - site.hostname - when DeployWww - deploy.fqdn - when DeployAlternativeDomain - deploy.hostname.dup.tap do |h| - h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}") - end - when DeployHiddenService - deploy.onion - end - end.compact + def destinations + @destinations ||= site.deploys.map(&:destination).compact.select do |d| + File.directory?(d) + end.map do |d| + File.realpath(d) + end.uniq end # Recolecta todas las URIs menos imágenes # + # TODO: Para los sitios con DeployLocalizedDomain estamos buscando + # URIs de más. + # # @return [Array] def uris - @uris ||= Dir.chdir destination do - (Dir.glob('**/*.html') + Dir.glob('public/**/*').reject do |p| - File.directory? p - end.reject do |p| - p = p.downcase + @uris ||= + destinations.map do |destination| + locales.map do |locale| + uri = "/#{locale}/".squeeze('/') + dir = File.join(destination, locale) - IMAGES.any? do |i| - p.end_with? i + next unless File.directory? dir + + files(dir).map do |f| + uri + f + end end - end).map do |uri| - "/#{uri}" - end + end.flatten(3).compact + end + + # @return [Array] + def locales + @locales ||= ['', site.locales.map(&:to_s)].flatten(1) + end + + # @param :dir [String] + # @return [Array] + def files(dir) + Dir.chdir(dir) do + pages = Dir.glob('**/*.html') + files = Dir.glob('public/**/*') + files = remove_directories files + files = remove_images files + + [pages, files].flatten(1) + end + end + + # @param :files [Array] + # @return [Array] + def remove_directories(files) + files.reject do |f| + File.directory? f + end + end + + def remove_images(files) + files.reject do |f| + IMAGES.include? File.extname(f).downcase end end end diff --git a/app/lib/action_dispatch/http/uploaded_file_decorator.rb b/app/lib/action_dispatch/http/uploaded_file_decorator.rb new file mode 100644 index 00000000..0bdebdc0 --- /dev/null +++ b/app/lib/action_dispatch/http/uploaded_file_decorator.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module ActionDispatch + module Http + # Normaliza los nombres de archivo para que se propaguen + # correctamente a través de todo el stack. + module UploadedFileDecorator + extend ActiveSupport::Concern + + included do + # Devolver el nombre de archivo con caracteres unicode + # normalizados + def original_filename + @original_filename.unicode_normalize + end + end + end + end +end + +ActionDispatch::Http::UploadedFile.include ActionDispatch::Http::UploadedFileDecorator diff --git a/app/lib/active_storage/attached/changes/create_one_decorator.rb b/app/lib/active_storage/attached/changes/create_one_decorator.rb new file mode 100644 index 00000000..bfb92478 --- /dev/null +++ b/app/lib/active_storage/attached/changes/create_one_decorator.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module ActiveStorage + module Attached::Changes::CreateOneDecorator + extend ActiveSupport::Concern + + included do + private + + # A partir de ahora todos los archivos se suben al servicio de + # cada sitio. + def attachment_service_name + record.name.to_sym + end + end + end +end + +ActiveStorage::Attached::Changes::CreateOne.include ActiveStorage::Attached::Changes::CreateOneDecorator diff --git a/app/lib/active_storage/service/jekyll_service.rb b/app/lib/active_storage/service/jekyll_service.rb new file mode 100644 index 00000000..92b26e0e --- /dev/null +++ b/app/lib/active_storage/service/jekyll_service.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module ActiveStorage + class Service + # Sube los archivos a cada repositorio y los agrega al LFS de su + # repositorio git. + # + # @todo: Implementar LFS. No nos gusta mucho la idea porque duplica + # el espacio en disco, pero es la única forma que tenemos (hasta que + # implementemos IPFS) para poder transferir los archivos junto con el + # sitio. + class JekyllService < Service::DiskService + # Genera un servicio para un sitio determinado + # + # @param :site [Site] + # @return [ActiveStorage::Service::JekyllService] + def self.build_for_site(site:) + new(root: File.join(site.path, 'public'), public: true).tap do |js| + js.name = site.name.to_sym + end + end + + # Lo mismo que en DiskService agregando el nombre de archivo en la + # firma. Esto permite que luego podamos guardar el archivo donde + # corresponde. + # + # @param :key [String] + # @param :expires_in [Integer] + # @param :content_type [String] + # @param :content_length [Integer] + # @param :checksum [String] + # @return [String] + def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) + instrument :url, key: key do |payload| + verified_token_with_expiration = ActiveStorage.verifier.generate( + { + key: key, + content_type: content_type, + content_length: content_length, + checksum: checksum, + service_name: name, + filename: filename_for(key) + }, + expires_in: expires_in, + purpose: :blob_token + ) + + generated_url = url_helpers.update_rails_disk_service_url(verified_token_with_expiration, host: current_host) + + payload[:url] = generated_url + + generated_url + end + end + + # Mantener retrocompatibilidad con cómo gestionamos los archivos + # subidos hasta ahora. + # + # @param :key [String] + # @return [String] + def folder_for(key) + key + end + + # Obtiene el nombre de archivo para esta key + # + # @param :key [String] + # @return [String] + def filename_for(key) + ActiveStorage::Blob.where(key: key).limit(1).pluck(:filename).first + end + + # Crea una ruta para la llave con un nombre conocido. + # + # @param :key [String] + # @return [String] + def path_for(key) + File.join root, folder_for(key), filename_for(key) + end + end + end +end diff --git a/app/lib/active_storage/service/registry_decorator.rb b/app/lib/active_storage/service/registry_decorator.rb new file mode 100644 index 00000000..c7096356 --- /dev/null +++ b/app/lib/active_storage/service/registry_decorator.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module ActiveStorage + class Service + # Modificaciones a ActiveStorage::Service::Registry + module RegistryDecorator + extend ActiveSupport::Concern + + included do + # El mismo comportamiento que #fetch con el agregado de generar + # un {JekyllService} para cada sitio. + def fetch(name) + services.fetch(name.to_sym) do |key| + if configurations.include?(key) + services[key] = configurator.build(key) + elsif (site = Site.find_by_name(key)) + services[key] = ActiveStorage::Service::JekyllService.build_for_site(site: site) + elsif block_given? + yield key + else + raise KeyError, "Missing configuration for the #{key} Active Storage service. " \ + "Configurations available for the #{configurations.keys.to_sentence} services." + end + end + end + end + end + end +end + +ActiveStorage::Service::Registry.include ActiveStorage::Service::RegistryDecorator diff --git a/app/models/active_storage/blob_decorator.rb b/app/models/active_storage/blob_decorator.rb new file mode 100644 index 00000000..9c01251a --- /dev/null +++ b/app/models/active_storage/blob_decorator.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module ActiveStorage + # Modificaciones a ActiveStorage::Blob + module BlobDecorator + extend ActiveSupport::Concern + + included do + # Permitir que llegue el nombre de archivo al servicio de subida de + # archivos. + # + # @return [Hash] + def service_metadata + if forcibly_serve_as_binary? + { content_type: ActiveStorage.binary_content_type, disposition: :attachment, filename: filename } + elsif !allowed_inline? + { content_type: content_type, disposition: :attachment, filename: filename } + else + { content_type: content_type, filename: filename } + end + end + end + end +end + +ActiveStorage::Blob.include ActiveStorage::BlobDecorator diff --git a/app/models/deploy_local.rb b/app/models/deploy_local.rb index 4fa588f5..1b661059 100644 --- a/app/models/deploy_local.rb +++ b/app/models/deploy_local.rb @@ -28,15 +28,17 @@ class DeployLocal < Deploy # Obtener el tamaño de todos los archivos y directorios (los # directorios son archivos :) def size - paths = [destination, File.join(destination, '**', '**')] + @size ||= begin + paths = [destination, File.join(destination, '**', '**')] - Dir.glob(paths).map do |file| - if File.symlink? file - 0 - else - File.size(file) - end - end.inject(:+) + Dir.glob(paths).map do |file| + if File.symlink? file + 0 + else + File.size(file) + end + end.inject(:+) + end end def destination diff --git a/app/models/deploy_rsync.rb b/app/models/deploy_rsync.rb new file mode 100644 index 00000000..996f8cdd --- /dev/null +++ b/app/models/deploy_rsync.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +# Sincroniza sitios a servidores remotos usando Rsync. El servidor +# remoto tiene que tener rsync instalado. +class DeployRsync < Deploy + store :values, accessors: %i[destination host_keys], coder: JSON + + def deploy + ssh? && rsync + end + + # El espacio remoto es el mismo que el local + # + # @return [Integer] + def size + deploy_local.size + end + + # Devolver el destino o lanzar un error si no está configurado + def destination + values[:destination].tap do |d| + raise(ArgumentError, 'destination no está configurado') if d.blank? + end + end + + private + + # Verificar la conexión SSH implementando Trust On First Use + # + # TODO: Medir el tiempo que tarda en iniciarse la conexión + # + # @return [Boolean] + def ssh? + user, host = user_host + ssh_available = false + + Net::SSH.start(host, user, verify_host_key: tofu, timeout: 5) do |ssh| + if values[:host_keys].blank? + # Guardar las llaves que se encontraron en la primera conexión + values[:host_keys] = ssh.transport.host_keys.map do |host_key| + "#{host_key.ssh_type} #{host_key.fingerprint}" + end + + ssh_available = save + else + ssh_available = true + end + end + + ssh_available + rescue Exception => e + ExceptionNotifier.notify_exception(e, data: { site: site.id, hostname: host, user: user }) + + false + end + + def env + { + 'HOME' => home_dir, + 'PATH' => '/usr/bin', + 'LANG' => ENV['LANG'] + } + end + + # Confiar en la primera llave que encontremos, fallar si cambian + # + # @return [Symbol] + def tofu + values[:host_keys].present? ? :always : :accept_new + end + + # Devuelve el par user host + # + # @return [Array] + def user_host + destination.split(':', 2).first.split('@', 2).tap do |d| + next unless d.size == 1 + + d.insert(0, nil) + end + end + + # Sincroniza hacia el directorio remoto + # + # @return [Boolean] + def rsync + run %(rsync -aviH --timeout=5 #{Shellwords.escape source}/ #{Shellwords.escape destination}/) + end + + # El origen es el destino de la compilación + # + # @return [String] + def source + deploy_local.destination + end + + def deploy_local + @deploy_local ||= site.deploys.find_by(type: 'DeployLocal') + end +end diff --git a/app/models/metadata_file.rb b/app/models/metadata_file.rb index 80cefa27..71d3f049 100644 --- a/app/models/metadata_file.rb +++ b/app/models/metadata_file.rb @@ -13,12 +13,18 @@ class MetadataFile < MetadataTemplate value == default_value end + # No hay valores sugeridos para archivos subidos. + # + # XXX: Esto ayuda a deserializar en {Site#everything_of} + def values; end + def validate super errors << I18n.t("metadata.#{type}.site_invalid") if site.invalid? errors << I18n.t("metadata.#{type}.path_required") if path_missing? errors << I18n.t("metadata.#{type}.no_file_for_description") if no_file_for_description? + errors << I18n.t("metadata.#{type}.attachment_missing") if path? && !static_file errors.compact! errors.empty? @@ -34,12 +40,6 @@ class MetadataFile < MetadataTemplate value['path'].is_a?(String) end - # Determina si la ruta es opcional pero deja pasar si la ruta se - # especifica - def path_optional? - !required && !path? - end - # Asociar la imagen subida al sitio y obtener la ruta # # XXX: Si evitamos guardar cambios con changed? no tenemos forma de @@ -48,13 +48,7 @@ class MetadataFile < MetadataTemplate # repetida. def save value['description'] = sanitize value['description'] - - if path? - hardlink - value['path'] = relative_destination_path - else - value['path'] = nil - end + value['path'] = static_file ? relative_destination_path_with_filename.to_s : nil true end @@ -71,101 +65,88 @@ class MetadataFile < MetadataTemplate # XXX: La última opción provoca archivos duplicados, pero es lo mejor # que tenemos hasta que resolvamos https://0xacab.org/sutty/sutty/-/issues/213 # - # @return [ActiveStorage::Attachment] + # @todo encontrar una forma de obtener el attachment sin tener que + # recurrir al último subido. + # + # @return [ActiveStorage::Attachment,nil] def static_file - return unless path? - @static_file ||= case value['path'] when ActionDispatch::Http::UploadedFile site.static_files.last if site.static_files.attach(value['path']) when String - if (blob = ActiveStorage::Blob.where(key: key_from_path).pluck(:id).first) - site.static_files.find_by(blob_id: blob) - elsif site.static_files.attach(io: path.open, filename: path.basename) - site.static_files.last + if (blob_id = ActiveStorage::Blob.where(key: key_from_path).pluck(:id).first) + site.static_files.find_by(blob_id: blob_id) + elsif path? && pathname.exist? && site.static_files.attach(io: pathname.open, filename: pathname.basename) + site.static_files.last.tap do |s| + s.blob.update(key: key_from_path) + end end end end + # Obtiene la ruta absoluta al archivo + # + # @return [Pathname] + def pathname + raise NoMethodError unless uploaded? + + @pathname ||= Pathname.new(File.join(site.path, value['path'])) + end + + # Obtiene la key del attachment a partir de la ruta + # + # @return [String] def key_from_path - path.dirname.basename.to_s + pathname.dirname.basename.to_s end def path? value['path'].present? end + def description? + value['description'].present? + end + private - def filemagic - @filemagic ||= FileMagic.new(FileMagic::MAGIC_MIME) - end - + # Obtener la ruta al archivo relativa al sitio + # # @return [Pathname] - def path - @path ||= Pathname.new(File.join(site.path, value['path'])) - end - - def file - return unless path? - - @file ||= - case value['path'] - when ActionDispatch::Http::UploadedFile then value['path'].tempfile.path - when String then File.join(site.path, value['path']) - end - end - - # Hacemos un link duro para colocar el archivo dentro del repositorio - # y no duplicar el espacio que ocupan. Esto requiere que ambos - # directorios estén dentro del mismo punto de montaje. - # - # XXX: Asumimos que el archivo destino no existe porque siempre - # contiene una key única. - # - # @return [Boolean] - def hardlink - return if hardlink? - return if File.exist? destination_path - - FileUtils.mkdir_p(File.dirname(destination_path)) - FileUtils.ln(uploaded_path, destination_path).zero? - end - - def hardlink? - File.stat(uploaded_path).ino == File.stat(destination_path).ino - rescue Errno::ENOENT - false - end - - # Obtener la ruta al archivo - # https://stackoverflow.com/a/53908358 - def uploaded_relative_path - ActiveStorage::Blob.service.path_for(static_file.key) - end - - # @return [String] - def uploaded_path - Rails.root.join uploaded_relative_path - end - - # La ruta del archivo mantiene el nombre original pero contiene el - # nombre interno y único del archivo para poder relacionarlo con el - # archivo subido en Sutty. - # - # @return [String] - def relative_destination_path - @relative_destination_path ||= File.join('public', static_file.key, static_file.filename.to_s) - end - - # @return [String] def destination_path - @destination_path ||= File.join(site.path, relative_destination_path) + Pathname.new(static_file_path) + end + + # Agrega el nombre de archivo a la ruta para tener retrocompatibilidad + # + # @return [Pathname] + def destination_path_with_filename + destination_path.realpath + # Si el archivo no llegara a existir, en lugar de hacer fallar todo, + # devolvemos la ruta original, que puede ser el archivo que no existe + # o vacía si se está subiendo uno. + rescue Errno::ENOENT => e + ExceptionNotifier.notify_exception(e) + + value['path'] + end + + def relative_destination_path_with_filename + destination_path_with_filename.relative_path_from(Pathname.new(site.path).realpath) + end + + def static_file_path + case static_file.blob.service.name + when :local + File.join(site.path, 'public', static_file.key, static_file.filename.to_s) + else + static_file.blob.service.path_for(static_file.key) + end end # No hay archivo pero se lo describió def no_file_for_description? - value['description'].present? && value['path'].blank? + !path? && description? end end diff --git a/app/models/metadata_image.rb b/app/models/metadata_image.rb index f91a6273..fc61a4d7 100644 --- a/app/models/metadata_image.rb +++ b/app/models/metadata_image.rb @@ -5,7 +5,7 @@ class MetadataImage < MetadataFile def validate super - errors << I18n.t('metadata.image.not_an_image') unless image? + errors << I18n.t('metadata.image.not_an_image') if path? && !image? errors.compact! errors.empty? @@ -13,8 +13,6 @@ class MetadataImage < MetadataFile # Determina si es una imagen def image? - return true unless file - - filemagic.file(file).starts_with? 'image/' + static_file&.blob&.send(:web_image?) end end diff --git a/app/models/metadata_markdown.rb b/app/models/metadata_markdown.rb index 1e8b4fc8..a09e351c 100644 --- a/app/models/metadata_markdown.rb +++ b/app/models/metadata_markdown.rb @@ -12,6 +12,6 @@ class MetadataMarkdown < MetadataText # markdown y se eliminan autolinks. Mejor es habilitar la generación # SAFE de CommonMark en la configuración del sitio. def sanitize(string) - string + string.unicode_normalize end end diff --git a/app/models/metadata_markdown_content.rb b/app/models/metadata_markdown_content.rb index 92a1ab21..75088e30 100644 --- a/app/models/metadata_markdown_content.rb +++ b/app/models/metadata_markdown_content.rb @@ -25,6 +25,6 @@ class MetadataMarkdownContent < MetadataText # markdown y se eliminan autolinks. Mejor es deshabilitar la # generación SAFE de CommonMark en la configuración del sitio. def sanitize(string) - string.tr("\r", '') + string.tr("\r", '').unicode_normalize end end diff --git a/app/models/metadata_permalink.rb b/app/models/metadata_permalink.rb index 59b68461..30ad32cc 100644 --- a/app/models/metadata_permalink.rb +++ b/app/models/metadata_permalink.rb @@ -19,7 +19,7 @@ class MetadataPermalink < MetadataString # puntos suspensivos, la primera / para que siempre sea relativa y # agregamos una / al final si la ruta no tiene extensión. def sanitize(value) - value = value.strip.gsub('..', '/').gsub('./', '').squeeze('/') + value = value.strip.unicode_normalize.gsub('..', '/').gsub('./', '').squeeze('/') value = value[1..-1] if value.start_with? '/' value += '/' if File.extname(value).blank? diff --git a/app/models/metadata_string.rb b/app/models/metadata_string.rb index 95aac4d4..c1d888b1 100644 --- a/app/models/metadata_string.rb +++ b/app/models/metadata_string.rb @@ -17,7 +17,7 @@ class MetadataString < MetadataTemplate def sanitize(string) return '' if string.blank? - sanitizer.sanitize(string.strip, + sanitizer.sanitize(string.strip.unicode_normalize, tags: [], attributes: []).strip.html_safe end diff --git a/app/models/metadata_template.rb b/app/models/metadata_template.rb index 26351249..c778e1b2 100644 --- a/app/models/metadata_template.rb +++ b/app/models/metadata_template.rb @@ -184,9 +184,12 @@ MetadataTemplate = Struct.new(:site, :document, :name, :label, :type, return if string.nil? return string unless string.is_a? String - sanitizer.sanitize(string.tr("\r", ''), - tags: allowed_tags, - attributes: allowed_attributes).strip.html_safe + sanitizer + .sanitize(string.tr("\r", '').unicode_normalize, + tags: allowed_tags, + attributes: allowed_attributes) + .strip + .html_safe end def sanitizer diff --git a/app/models/post/template_field.rb b/app/models/post/template_field.rb deleted file mode 100644 index 08da933b..00000000 --- a/app/models/post/template_field.rb +++ /dev/null @@ -1,360 +0,0 @@ -# frozen_string_literal: true - -# Representa los distintos tipos de campos que pueden venir de una -# plantilla compleja -class Post - class TemplateField - attr_reader :post, :contents, :key - - STRING_VALUES = %w[string text url number email password date year - image video audio document].freeze - - # Tipo de valores que son archivos - FILE_TYPES = %w[image video audio document].freeze - - def initialize(post, key, contents) - @post = post - @key = key - @contents = contents - end - - def title - contents.dig('title') if complex? - end - - def subtitle - contents.dig('subtitle') if complex? - end - - # Obtiene el valor - def value - complex? ? contents.dig('value') : contents - end - - def max - return 0 if simple? - - contents.fetch('max', 0) - end - - def min - return 0 if simple? - - contents.fetch('min', 0) - end - - # TODO: volver elegante! - def type - return @type if @type - - if image? - @type = 'image' - elsif email? - @type = 'email' - elsif url? - @type = 'url' - elsif number? - @type = 'number' - elsif password? - @type = 'password' - elsif date? - @type = 'date' - elsif year? - @type = 'year' - elsif text_area? - @type = 'text_area' - elsif check_box_group? - @type = 'check_box_group' - elsif radio_group? - @type = 'radio_group' - elsif string? - @type = 'text' - # TODO: volver a hacer funcionar esto y ahorranos los multiple: - # false - elsif string? && contents.split('/', 2).count == 2 - @type = 'select' - elsif nested? - @type = 'table' - elsif array? - @type = 'select' - elsif boolean? - @type = 'check_box' - end - - @type - end - - # Devuelve los valores vacíos según el tipo - def empty_value - if string? - '' - elsif nested? - # TODO: devolver las keys también - {} - elsif array? - [] - elsif boolean? - false - end - end - - def cols - complex? && contents.dig('cols') - end - - def align - complex? && contents.dig('align') - end - - # El campo es requerido si es complejo y se especifica que lo sea - def required? - complex? && contents.dig('required') - end - - def boolean? - value.is_a?(FalseClass) || value.is_a?(TrueClass) - end - - def string? - value.is_a? String - end - - def text_area? - value == 'text' - end - - def url? - value == 'url' - end - - def email? - value == 'email' || value == 'mail' - end - alias mail? email? - - def date? - value == 'date' - end - - def password? - value == 'password' - end - - def number? - value == 'number' - end - - def year? - value == 'year' - end - - def file? - string? && FILE_TYPES.include?(value) - end - - def image? - array? ? value.first == 'image' : value == 'image' - end - - # Si la plantilla es simple no está admitiendo Hashes como valores - def simple? - !complex? - end - - def complex? - contents.is_a? Hash - end - - # XXX Retrocompatibilidad - def to_s - key - end - - # Convierte el campo en un parámetro - def to_param - if nested? - { key.to_sym => {} } - elsif array? && multiple? - { key.to_sym => [] } - else - key.to_sym - end - end - - # Convierte la plantilla en el formato de front_matter - def to_front_matter - { key => empty_value } - end - - def check_box_group? - array? && (complex? && contents.fetch('checkbox', false)) - end - - def radio_group? - array? && (complex? && contents.fetch('radio', false)) - end - - def array? - value.is_a? Array - end - - # TODO: detectar cuando es complejo y tomar el valor de :multiple - def multiple? - # si la plantilla es simple, es multiple cuando tenemos un array - return array? if simple? - - array? && contents.fetch('multiple', true) - end - - # Detecta si el valor es una tabla de campos - def nested? - value.is_a?(Hash) || (array? && value.first.is_a?(Hash)) - end - - # Un campo acepta valores abiertos si no es un array con múltiples - # elementos - def open? - # Todos los valores simples son abiertos - return true unless complex? - return false unless array? - - # La cosa se complejiza cuando tenemos valores complejos - # - # Si tenemos una lista cerrada de valores, necesitamos saber si el - # campo es abierto o cerrado. Si la lista tiene varios elementos, - # es una lista cerrada, opcionalmente abierta. Si la lista tiene - # un elemento, quiere decir que estamos autocompletando desde otro - # lado. - contents.fetch('open', value.count < 2) - end - - def closed? - !open? - end - - # Determina si los valores del campo serán públicos después - # - # XXX Esto es solo una indicación, el theme Jekyll tiene que - # respetarlos por su lado luego - def public? - # Todos los campos son públicos a menos que se indique lo - # contrario - simple? || contents.fetch('public', true) - end - - def private? - !public? - end - - def human - h = key.humanize - - h - end - - def label - h = (complex? && contents.dig('label')) || human - h += ' *' if required? - - h - end - - def help - complex? && contents.dig('help') - end - - def nested_fields - return unless nested? - - v = value - v = value.first if array? - - @nested_fields ||= v.map do |k, sv| - Post::TemplateField.new post, k, sv - end - end - - # Obtiene los valores posibles para el campo de la plantilla - def values - return 'false' if value == false - return 'true' if value == true - # XXX por alguna razón `value` no refiere a value() :/ - return '' if STRING_VALUES.include? value - # Las listas cerradas no necesitan mayor procesamiento - return value if array? && closed? && value.count > 1 - # Y las vacías tampoco - return value if array? && value.empty? - # Ahorrarnos el trabajo - return @values if @values - - # Duplicar el valor para no tener efectos secundarios luego (?) - value = self.value.dup - - # Para obtener los valores posibles, hay que procesar la string y - # convertirla a parametros - - # Si es una array de un solo elemento, es un indicador de que - # tenemos que rellenarla con los valores que indica. - # - # El primer valor es el que trae la string de autocompletado - values = array? ? value.shift : value - - # Si el valor es un array con más de un elemento, queremos usar - # esas opciones. Pero si además es abierto, queremos traer los - # valores cargados anteriormente. - - # Procesamos el valor, buscando : como separador de campos que - # queremos encontrar y luego los unimos - _value = (values&.split(':', 2) || []).map do |v| - # Tenemos hasta tres niveles de búsqueda - collection, attr, subattr = v.split('/', 3) - - if collection == 'site' - # TODO: puede ser peligroso permitir acceder a cualquier - # atributo de site? No estamos trayendo nada fuera de - # lo normal - post.site.send(attr.to_sym) - # Si hay un subatributo, tenemos que averiguar todos los - # valores dentro de el - # TODO volver elegante! - # TODO volver recursivo! - elsif subattr - post.site.everything_of(attr, lang: collection) - .compact - .map { |sv| sv[subattr] } - .flatten - .compact - .uniq - else - post.site.everything_of(attr, lang: collection).compact - end - end - - # Si el valor es abierto, sumar los valores auto-completados a - # lo pre-cargados. - # - # En este punto _value es un array de 1 o 2 arrays, si es de uno, - # value tambien tiene que serlo. Si es de 2, hay que unir cada - # una - if open? - if _value.count == 1 - _value = [(_value.first + value).uniq] - elsif _value.count == 2 - _value = _value.each_with_index.map do |v, i| - v + value.fetch(i, []) - end - end - end - - # Crea un array de arrays, útil para los select - # [ [ 1, a ], [ 2, b ] ] - # aunque si no hay un : en el autocompletado, el array queda - # [ [ 1, 1 ], [ 2, 2 ] ] - values = _value.empty? ? [] : _value.last.zip(_value.first) - - # En última instancia, traer el valor por defecto y ahorrarnos - # volver a procesar - @values = values - end - end -end diff --git a/app/models/site.rb b/app/models/site.rb index 5b78d625..fd318995 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -57,7 +57,7 @@ class Site < ApplicationRecord # Carga el sitio Jekyll una vez que se inicializa el modelo o después # de crearlo after_initialize :load_jekyll - after_create :load_jekyll, :static_file_migration! + after_create :load_jekyll # Cambiar el nombre del directorio before_update :update_name! before_save :add_private_key_if_missing! @@ -101,6 +101,26 @@ class Site < ApplicationRecord "https://#{hostname}#{slash ? '/' : ''}" end + # TODO: Cambiar al mergear origin-referer + # + # @return [Array] + def hostnames + @hostnames ||= deploys.map do |deploy| + case deploy + when DeployLocal + hostname + when DeployWww + deploy.fqdn + when DeployAlternativeDomain + deploy.hostname.dup.tap do |h| + h.replace(h.end_with?('.') ? h[0..-2] : "#{h}.#{Site.domain}") + end + when DeployHiddenService + deploy.onion + end + end.compact + end + # Obtiene los dominios alternativos # # @return Array @@ -123,7 +143,9 @@ class Site < ApplicationRecord # # @return Array def urls(slash: true) - alternative_urls(slash: slash) << url(slash: slash) + @urls ||= hostnames.map do |h| + "https://#{h}#{slash ? '/' : ''}" + end end def invitade?(usuarie) @@ -166,8 +188,10 @@ class Site < ApplicationRecord end # Similar a site.i18n en jekyll-locales + # + # @return [Hash] def i18n - data[I18n.locale.to_s] + data[I18n.locale.to_s] || {} end # Devuelve el idioma por defecto del sitio, el primero de la lista. @@ -470,15 +494,10 @@ class Site < ApplicationRecord config.theme = design.gem unless design.no_theme? config.description = description config.title = title - config.url = url + config.url = url(slash: false) config.hostname = hostname end - # Migra los archivos a Sutty - def static_file_migration! - Site::StaticFileMigration.new(site: self).migrate! - end - # Valida si el sitio tiene al menos una forma de alojamiento asociada # y es la local # diff --git a/app/models/site/static_file_migration.rb b/app/models/site/static_file_migration.rb deleted file mode 100644 index 36a882bf..00000000 --- a/app/models/site/static_file_migration.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -class Site - # Obtiene todos los archivos relacionados en artículos del sitio y los - # sube a Sutty. - class StaticFileMigration - # Tipos de metadatos que contienen archivos - STATIC_TYPES = %i[file image].freeze - - attr_reader :site - - def initialize(site:) - @site = site - end - - def migrate! - modified = site.docs.map do |doc| - next unless STATIC_TYPES.map do |field| - next unless doc.attribute? field - next unless doc[field].path? - next unless doc[field].static_file - - true - end.any? - - log.write "#{doc.path.relative};no se pudo guardar\n" unless doc.save(validate: false) - - doc.path.absolute - end.compact - - log.close - - return if modified.empty? - - # TODO: Hacer la migración desde el servicio de creación de sitios? - site.repository.commit(file: modified, - message: I18n.t('sites.static_file_migration'), - usuarie: author) - end - - private - - def author - @author ||= GitAuthor.new email: "sutty@#{Site.domain}", - name: 'Sutty' - end - - def log - @log ||= File.open(File.join(site.path, 'migration.csv'), 'w') - end - end -end diff --git a/app/models/site_blazer.rb b/app/models/site_blazer.rb new file mode 100644 index 00000000..76dee12a --- /dev/null +++ b/app/models/site_blazer.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +SiteBlazer = Struct.new(:site) diff --git a/app/models/stat.rb b/app/models/stat.rb index 5f72ccd0..6c681aa4 100644 --- a/app/models/stat.rb +++ b/app/models/stat.rb @@ -3,8 +3,16 @@ # Registran cuándo fue la última recolección de datos. class Stat < ApplicationRecord # XXX: Los intervalos van en orden de mayor especificidad a menor - INTERVALS = %i[day month year].freeze + INTERVALS = %i[day].freeze RESOURCES = %i[builds space_used build_time].freeze + COLUMNS = %i[http_referer geoip2_data_country_name].freeze belongs_to :site + + # El intervalo por defecto + # + # @return [Symbol] + def self.default_interval + INTERVALS.first + end end diff --git a/app/models/usuarie.rb b/app/models/usuarie.rb index 6de7ba4b..c88dcc68 100644 --- a/app/models/usuarie.rb +++ b/app/models/usuarie.rb @@ -11,6 +11,8 @@ class Usuarie < ApplicationRecord has_many :roles has_many :sites, through: :roles + has_many :blazer_audits, foreign_key: 'user_id', class_name: 'Blazer::Audit' + has_many :blazer_queries, foreign_key: 'creator_id', class_name: 'Blazer::Query' def name email.split('@', 2).first diff --git a/app/policies/site_blazer_policy.rb b/app/policies/site_blazer_policy.rb new file mode 100644 index 00000000..a6ea01b7 --- /dev/null +++ b/app/policies/site_blazer_policy.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +# Les invitades no pueden ver las estadísticas (aun) +SiteBlazerPolicy = Struct.new(:usuarie, :site_blazer) do + def home? + site_blazer&.site&.usuarie? usuarie + end + + alias_method :show?, :home? +end diff --git a/app/services/site_service.rb b/app/services/site_service.rb index 5e2fc706..22423bb8 100644 --- a/app/services/site_service.rb +++ b/app/services/site_service.rb @@ -9,6 +9,7 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do self.site = Site.new params add_role temporal: false, rol: 'usuarie' + sync_nodes I18n.with_locale(usuarie&.lang&.to_sym || I18n.default_locale) do site.save && @@ -144,4 +145,11 @@ SiteService = Struct.new(:site, :usuarie, :params, keyword_init: true) do PostService.new(site: site, usuarie: usuarie, post: post, params: params).update end + + # Crea los deploys necesarios para sincronizar a otros nodos de Sutty + def sync_nodes + Rails.application.nodes.each do |node| + site.deploys.build(type: 'DeployRsync', destination: "sutty@#{node}:#{site.hostname}") + end + end end diff --git a/app/views/blazer/check_mailer/failing_checks.haml b/app/views/blazer/check_mailer/failing_checks.haml new file mode 100644 index 00000000..c28c3936 --- /dev/null +++ b/app/views/blazer/check_mailer/failing_checks.haml @@ -0,0 +1,5 @@ +%ul + - @checks.each do |check| + %li + = check.query.name + = check.state diff --git a/app/views/blazer/check_mailer/state_change.haml b/app/views/blazer/check_mailer/state_change.haml new file mode 100644 index 00000000..48418a58 --- /dev/null +++ b/app/views/blazer/check_mailer/state_change.haml @@ -0,0 +1,30 @@ +!!! +%html + %head + %meta{:content => "text/html; charset=UTF-8", "http-equiv" => "Content-Type"}/ + %body{:style => "font-family: 'Helvetica Neue', Arial, Helvetica; font-size: 14px; color: #333;"} + - if @error + %p= @error + - elsif @rows_count > 0 && @check_type == "bad_data" + %p + - if @rows_count <= 10 + = pluralize(@rows_count, "row") + - else + Showing 10 of #{@rows_count} rows + %table{:style => "width: 100%; border-spacing: 0; border-collapse: collapse;"} + %thead + %tr + - @columns.first(5).each do |column| + %th{:style => "padding: 8px; line-height: 1.4; text-align: left; vertical-align: bottom; border-bottom: 2px solid #ddd; width: #{(100 / @columns.size).round(2)}%;"} + = column + %tbody + - @rows.first(10).each do |row| + %tr + - @columns.first(5).each_with_index do |column, i| + %td{:style => "padding: 8px; line-height: 1.4; vertical-align: top; border-top: 1px solid #ddd;"} + - value = row[i] + - if @column_types[i] == "time" && value.to_s.length > 10 + - value = Time.parse(value).in_time_zone(Blazer.time_zone) rescue value + = value + - if @columns.size > 5 + %p{:style => "color: #999;"} Only first 5 columns shown diff --git a/app/views/blazer/queries/home.haml b/app/views/blazer/queries/home.haml new file mode 100644 index 00000000..977b6bda --- /dev/null +++ b/app/views/blazer/queries/home.haml @@ -0,0 +1,9 @@ +#queries + %table.table + %tbody.list + - @queries.each do |query| + %tr + -# + Por alguna razón no tenemos acceso a query_path para poder + generar la URL según Rails + %td= link_to query[:name], "/sites/#{params[:site_id]}/stats/queries/#{query.to_param}" diff --git a/app/views/blazer/queries/show.haml b/app/views/blazer/queries/show.haml new file mode 100644 index 00000000..3b5cb152 --- /dev/null +++ b/app/views/blazer/queries/show.haml @@ -0,0 +1,51 @@ +- blazer_title @query.name +.container + .row + .col-12 + %h1= @query.name + - if @query.description.present? + %p.lead= @query.description + - unless @result.chart_type.blank? + .col-12 + - case @result.chart_type + - when 'line' + = line_chart @chart_data, **@chart_options + - when 'line2' + = line_chart @chart_data, **@chart_options + - when 'pie' + = pie_chart @chart_data, **@chart_options + - when 'bar' + = column_chart @chart_data, **@chart_options + - when 'bar2' + = column_chart @chart_data, **@chart_options + - when 'scatter' + = scatter_chart @chart_data, **@chart_options + .col-12 + %table.table + %thead + %tr + - @result.columns.each do |key| + - next if key.include? 'ciphertext' + - next if key.include? 'encrypted' + %th.position-sticky.background-white{ style: 'top: 0' }= t("blazer.columns.#{key}", default: key.titleize) + %tbody + - @result.rows.each do |row| + %tr + - row.each_with_index do |v, i| + - k = @result.columns[i] + - next if k.include? 'ciphertext' + - next if k.include? 'encrypted' + %td + - if v.is_a?(Time) + - v = blazer_time_value(@data_source, k, v) + + - unless v.nil? + - if v.is_a?(String) && v.empty? + %span.text-muted= t('.empty') + - elsif @data_source.linked_columns[k] + = link_to blazer_format_value(k, v), @data_source.linked_columns[k].gsub('{value}', u(v.to_s)), target: '_blank' + - else + = blazer_format_value(k, v) + + - if (v2 = (@result.boom[k] || {})[v.nil? ? v : v.to_s]) + %span.text-muted= v2 diff --git a/app/views/deploys/_deploy_rsync.haml b/app/views/deploys/_deploy_rsync.haml new file mode 100644 index 00000000..0aab9802 --- /dev/null +++ b/app/views/deploys/_deploy_rsync.haml @@ -0,0 +1 @@ +-# nada diff --git a/app/views/layouts/_breadcrumb.haml b/app/views/layouts/_breadcrumb.haml index c4920bc7..dc0e3158 100644 --- a/app/views/layouts/_breadcrumb.haml +++ b/app/views/layouts/_breadcrumb.haml @@ -12,7 +12,7 @@ - else %span.line-clamp-1= link_to crumb.name, crumb.url - - if current_usuarie + - if @current_usuarie || current_usuarie %ul.navbar-nav - if @site&.tienda? %li.nav-item @@ -20,5 +20,5 @@ role: 'button', class: 'btn' %li.nav-item - = link_to t('.logout'), destroy_usuarie_session_path, + = link_to t('.logout'), main_app.destroy_usuarie_session_path, method: :delete, role: 'button', class: 'btn' diff --git a/app/views/layouts/blazer/application.haml b/app/views/layouts/blazer/application.haml new file mode 100644 index 00000000..add94190 --- /dev/null +++ b/app/views/layouts/blazer/application.haml @@ -0,0 +1,14 @@ +!!! +%html + %head + %meta{content: 'text/html; charset=UTF-8', 'http-equiv': 'Content-Type'}/ + %title= blazer_title ? blazer_title : 'Sutty' + %meta{charset: 'utf-8'}/ + = favicon_link_tag 'blazer/favicon.png' + = stylesheet_link_tag 'application' + = javascript_pack_tag 'blazer', 'data-turbolinks-track': 'reload' + = csrf_meta_tags + %body{ class: yield(:body) } + .container-fluid#sutty + = render 'layouts/breadcrumb' + = yield diff --git a/app/views/posts/attributes/_file.haml b/app/views/posts/attributes/_file.haml index 19175b9c..54f9f81a 100644 --- a/app/views/posts/attributes/_file.haml +++ b/app/views/posts/attributes/_file.haml @@ -1,7 +1,5 @@ .form-group - if metadata.static_file - = hidden_field_tag "#{base}[#{attribute}][path]", metadata.value['path'] - - case metadata.static_file.blob.content_type - when %r{\Avideo/} = video_tag url_for(metadata.static_file), @@ -14,13 +12,17 @@ - else = link_to t('posts.attribute_ro.file.download'), url_for(metadata.static_file) - .custom-control.custom-switch - = check_box_tag "#{base}[#{attribute}][path]", '', false, id: "#{base}_#{attribute}_destroy", class: 'custom-control-input' - = label_tag "#{base}_#{attribute}_destroy", t('posts.attributes.file.destroy'), class: 'custom-control-label' + -# Mantener el valor si no enviamos ninguna imagen + = hidden_field_tag "#{base}[#{attribute}][path]", metadata.value['path'] + -# Los archivos requeridos solo se pueden reemplazar + - unless metadata.required + .custom-control.custom-switch + = check_box_tag "#{base}[#{attribute}][path]", '', false, id: "#{base}_#{attribute}_destroy", class: 'custom-control-input' + = label_tag "#{base}_#{attribute}_destroy", t('posts.attributes.file.destroy'), class: 'custom-control-label' .custom-file = file_field(*field_name_for(base, attribute, :path), - **field_options(attribute, metadata), + **field_options(attribute, metadata, required: (metadata.required && !metadata.path?)), class: "custom-file-input #{invalid(post, attribute)}", data: { preview: "#{attribute}-preview" }) = label_tag "#{base}_#{attribute}_path", @@ -30,7 +32,7 @@ .form-group = label_tag "#{base}_#{attribute}_description", - post_label_t(attribute, :description, post: post) + post_label_t(attribute, :description, post: post, required: false) = text_field(*field_name_for(base, attribute, :description), value: metadata.value['description'], dir: dir, lang: locale, diff --git a/app/views/posts/attributes/_image.haml b/app/views/posts/attributes/_image.haml index c11f4196..f4d9bb3d 100644 --- a/app/views/posts/attributes/_image.haml +++ b/app/views/posts/attributes/_image.haml @@ -1,5 +1,5 @@ .form-group - - if metadata.uploaded? + - if metadata.static_file = image_tag url_for(metadata.static_file), alt: metadata.value['description'], class: 'img-fluid', @@ -37,4 +37,3 @@ **field_options(attribute, metadata, required: false)) = render 'posts/attribute_feedback', post: post, attribute: [attribute, :description], metadata: metadata - diff --git a/app/views/posts/index.haml b/app/views/posts/index.haml index 90d65670..bc5c826c 100644 --- a/app/views/posts/index.haml +++ b/app/views/posts/index.haml @@ -15,6 +15,9 @@ - else %td= link_to t('posts.filter'), site_posts_path(@site, **@filter_params.merge(layout: layout.value)), class: 'btn btn-secondary btn-sm' + - if policy(@site_stat).index? + = link_to t('stats.index.title'), site_stats_path(@site), class: 'btn' + - if policy(@site).edit? = link_to t('sites.edit.btn', site: @site.title), edit_site_path(@site), class: 'btn' @@ -71,8 +74,8 @@ %table.table{ data: { controller: 'reorder' } } %caption.sr-only= t('posts.caption') %thead - %tr - %th.border-0.background-white.position-sticky{ style: 'top: 0; z-index: 2', colspan: '4' } + %tr.sticky-top + %th.border-0{ colspan: '4' } .d-flex.flex-row.justify-content-between %div = submit_tag t('posts.reorder.submit'), class: 'btn' diff --git a/app/views/stats/index.haml b/app/views/stats/index.haml index bfcf33ef..49fbd023 100644 --- a/app/views/stats/index.haml +++ b/app/views/stats/index.haml @@ -6,32 +6,57 @@ %p %small = t('.last_update') - %time{ datetime: @last_stat.created_at } - #{time_ago_in_words @last_stat.created_at}. + %time{ datetime: @last_stat.updated_at } + #{time_ago_in_words @last_stat.updated_at}. - .mb-5 + %form.mb-5.form-inline{ method: 'get' } - Stat::INTERVALS.each do |interval| - = link_to t(".#{interval}"), site_stats_path(interval: interval, urls: params[:urls]), class: "btn #{'btn-primary active' if @interval == interval}" + = link_to t(".#{interval}"), site_stats_path(interval: interval, urls: params[:urls], period_start: params[:period_start].to_date.try(:"beginning_of_#{interval}").to_date, period_end: params[:period_end]), class: "mb-0 btn #{'btn-primary active' if @interval == interval}" + + %input.form-control{ type: 'date', name: :period_start, value: params[:period_start] } + %input.form-control{ type: 'date', name: :period_end, value: params[:period_end] } + %button.btn.mb-0{ type: 'submit' }= t('.filter') .mb-5 %h2= t('.host.title', count: @hostnames.size) %p.lead= t('.host.description') = line_chart site_stats_host_path(@chart_params), **@chart_options - .mb-5 + #custom-urls.mb-5 %h2= t('.urls.title') %p.lead= t('.urls.description') - %form + %form{ method: 'get', action: '#custom-urls' } %input{ type: 'hidden', name: 'interval', value: @interval } .form-group %label{ for: 'urls' }= t('.urls.label') - %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size, aria_describedby: 'help-urls' }= @normalized_urls.join("\n") + %textarea#urls.form-control{ name: 'urls', autocomplete: 'on', required: true, rows: @normalized_urls.size + 1, aria_describedby: 'help-urls' }= @normalized_urls.join("\n") %small#help-urls.feedback.form-text.text-muted= t('.urls.help') .form-group %button.btn{ type: 'submit' }= t('.urls.submit') - if @normalized_urls.present? - = line_chart site_stats_uris_path(urls: params[:urls], **@chart_params), **@chart_options + = line_chart site_stats_uris_path(urls: @normalized_urls, **@chart_params), **@chart_options + .row.mb-5.row-cols-1.row-cols-md-2 + - @columns.each_pair do |column, values| + - next if values.blank? + .col.mb-5 + %h2= t(".columns.#{column}.title") + %p.lead= t(".columns.#{column}.description") + + %table.table + %colgroup + %col + %col + %thead + %tr.sticky-top + %th{ scope: 'col' }= t(".columns.#{column}.column") + %th{ scope: 'col' }= t('.columns.visits') + %tfoot + %tbody + - values.each_pair do |col, val| + %tr + %th{ scope: 'row', style: 'word-break: break-all' }= col.blank? ? t(".columns.#{column}.empty") : col + %td= val .mb-5 %h2= t('.resources.title') %p.lead= t('.resources.description') diff --git a/bin/access_logs b/bin/access_logs new file mode 100755 index 00000000..cfeeb57a --- /dev/null +++ b/bin/access_logs @@ -0,0 +1,15 @@ +#!/bin/sh +set -e + +# Volcar y eliminar todos los access logs de dos días atrás +date="`dateadd today -1d`" +file="/srv/http/_storage/${date}.psql.gz" +test -n "${date}" +test ! -f "${file}" + +psql -h postgresql "${DATABASE:-sutty}" sutty < "${file}" +begin; +copy (select * from access_logs where created_at < '${date}') to stdout; +delete from access_logs where created_at < '${date}'; +commit; +SQL diff --git a/config/application.rb b/config/application.rb index 7326ae0f..97ab244c 100644 --- a/config/application.rb +++ b/config/application.rb @@ -38,6 +38,13 @@ module Sutty config.active_storage.variant_processor = :vips + config.to_prepare do + # Load application's model / class decorators + Dir.glob(File.join(File.dirname(__FILE__), '..', 'app', '**', '*_decorator.rb')).sort.each do |c| + Rails.configuration.cache_classes ? require(c) : load(c) + end + end + config.after_initialize do ActiveStorage::DirectUploadsController.include ActiveStorage::AuthenticatedDirectUploadsController @@ -49,5 +56,9 @@ module Sutty EmailAddress::Config.error_messages translations.transform_keys(&:to_s), locale.to_s end end + + def nodes + @nodes ||= ENV.fetch('SUTTY_NODES', '').split(',') + end end end diff --git a/config/blazer.yml b/config/blazer.yml index 2ba0965f..11792ff1 100644 --- a/config/blazer.yml +++ b/config/blazer.yml @@ -50,7 +50,7 @@ user_method: current_usuarie user_name: email # custom before_action to use for auth -# before_action_method: require_admin +before_action_method: require_usuarie # email to send checks from from_email: blazer@<%= ENV.fetch('SUTTY', 'sutty.nl') %> diff --git a/config/initializers/analyze_job.rb b/config/initializers/analyze_job.rb deleted file mode 100644 index f268e0dd..00000000 --- a/config/initializers/analyze_job.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -# TODO: Estamos procesando el análisis de los archivos en el momento -# porque queremos obtener la ruta del archivo en el momento y no -# después. Necesitaríamos poder generar el vínculo en el -# repositorio a destiempo, modificando el Job de ActiveStorage -ActiveStorage::AnalyzeJob.queue_adapter = :inline diff --git a/config/locales/en.yml b/config/locales/en.yml index 99fbc8b1..10a4793b 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -41,10 +41,12 @@ en: not_an_image: 'Not an image' path_required: 'Missing image for upload' no_file_for_description: "Description with no associated image" + attachment_missing: "I couldn't save the image :(" file: site_invalid: 'The file cannot be stored if the site configuration is not valid' path_required: "Missing file for upload" no_file_for_description: "Description with no associated file" + attachment_missing: "I couldn't save the file :(" event: zone_missing: 'Inexistent timezone' date_missing: 'Event date is required' @@ -100,6 +102,10 @@ en: title: Alternative domain name success: Success! error: Error + deploy_rsync: + title: Synchronize to backup server + success: Success! + error: Error help: You can contact us by replying to this e-mail maintenance_mailer: notice: @@ -252,7 +258,7 @@ en: help: | These statistics show information about how your site is generated and how many resources it uses. - last_update: 'Updated every hour. Last update on ' + last_update: 'Updated daily. Last update on ' empty: 'There is no enough information yet. We invite you to come back in %{please_return_at}!' loading: 'Loading...' hour: 'Hourly' @@ -283,7 +289,19 @@ en: description: 'Average storage space used by your site.' build_time: title: 'Publication time' - description: 'Average time your site takes to build.' + description: 'Average time your site takes to build, from pressing "Publish changes" to actually being available on your site.' + columns: + visits: "Visits" + http_referer: + title: "Referers" + description: "Visits by origin" + column: "Referer" + empty: "(direct visit)" + geoip2_data_country_name: + title: "Countries" + description: "Visits by country" + column: "Country" + empty: "(couldn't detect country)" sites: donations: url: 'https://donaciones.sutty.nl/en/' @@ -610,3 +628,14 @@ en: edit: 'Editing' usuaries: index: 'Users' + stats: + index: 'Statistics' + blazer: + columns: + total: 'Total' + dia: 'Date' + date: 'Date' + visitas: 'Visits' + queries: + show: + empty: '(empty)' diff --git a/config/locales/es.yml b/config/locales/es.yml index 3ed72d63..02973de5 100644 --- a/config/locales/es.yml +++ b/config/locales/es.yml @@ -41,10 +41,12 @@ es: not_an_image: 'No es una imagen' path_required: 'Se necesita una imagen' no_file_for_description: 'Se envió una descripción sin imagen asociada' + attachment_missing: 'no pude guardar el archivo :(' file: site_invalid: 'El archivo no se puede almacenar si la configuración del sitio no es válida' path_required: 'Se necesita un archivo' no_file_for_description: 'se envió una descripción sin archivo asociado' + attachment_missing: 'no pude guardar el archivo :(' event: zone_missing: 'El huso horario no es correcto' date_missing: 'La fecha es obligatoria' @@ -100,6 +102,10 @@ es: title: Dominio alternativo success: ¡Éxito! error: Hubo un error + deploy_rsync: + title: Sincronizar al servidor alternativo + success: ¡Éxito! + error: Hubo un error help: Por cualquier duda, responde este correo para contactarte con nosotres. maintenance_mailer: notice: @@ -257,7 +263,7 @@ es: help: | Las estadísticas visibilizan información sobre cómo se genera y cuántos recursos utiliza tu sitio. - last_update: 'Actualizadas cada hora. Última actualización hace ' + last_update: 'Actualizadas diariamente. Última actualización hace ' empty: 'Todavía no hay información suficiente. Te invitamos a volver en %{please_return_at} :)' loading: 'Cargando...' hour: 'Por hora' @@ -288,7 +294,19 @@ es: description: 'Espacio en disco que ocupa en promedio tu sitio.' build_time: title: 'Tiempo de publicación' - description: 'Tiempo promedio que toma en publicarse tu sitio.' + description: 'Tiempo que tarda el sitio en generarse, desde que usas el botón "Publicar cambios" hasta que los puedes ver en el sitio' + columns: + visits: "Visitas" + http_referer: + title: "Referencias" + description: "Orígenes de las visitas" + column: "Referencia" + empty: "(visita directa)" + geoip2_data_country_name: + title: "Países" + description: "Cantidad de visitas por país" + column: "País" + empty: "(no se pudo detectar el país)" sites: donations: url: 'https://donaciones.sutty.nl/' @@ -618,3 +636,14 @@ es: edit: 'Editando' usuaries: index: 'Usuaries' + stats: + index: 'Estadísticas' + blazer: + columns: + total: 'Total' + dia: 'Fecha' + date: 'Fecha' + visitas: 'Visitas' + queries: + show: + empty: '(vacío)' diff --git a/config/routes.rb b/config/routes.rb index 2aa0056f..8bab18af 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -4,8 +4,6 @@ Rails.application.routes.draw do devise_for :usuaries get '/.well-known/change-password', to: redirect('/usuaries/edit') - mount Blazer::Engine, at: 'blazer' - root 'application#index' constraints(Constraints::ApiSubdomain.new) do diff --git a/db/migrate/20200206163257_install_blazer.rb b/db/migrate/20200206163257_install_blazer.rb index 9b084169..32799053 100644 --- a/db/migrate/20200206163257_install_blazer.rb +++ b/db/migrate/20200206163257_install_blazer.rb @@ -3,8 +3,6 @@ # Blazer class InstallBlazer < ActiveRecord::Migration[6.0] def change - return unless Rails.env.production? - create_table :blazer_queries do |t| t.references :creator t.string :name diff --git a/db/migrate/20211022224008_add_site_to_stats.rb b/db/migrate/20211022224008_add_site_to_stats.rb new file mode 100644 index 00000000..db2b43ab --- /dev/null +++ b/db/migrate/20211022224008_add_site_to_stats.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +# La recolección de estadísticas podría pertenecer a un sitio +class AddSiteToStats < ActiveRecord::Migration[6.1] + def change + add_belongs_to :stats, :site, index: true, null: true + end +end diff --git a/db/migrate/20211022225449_add_name_to_stats.rb b/db/migrate/20211022225449_add_name_to_stats.rb new file mode 100644 index 00000000..89a17ee0 --- /dev/null +++ b/db/migrate/20211022225449_add_name_to_stats.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +# Agregarle un nombre a la estadística +class AddNameToStats < ActiveRecord::Migration[6.1] + def change + add_column :stats, :name, :string, null: false + add_index :stats, :name, using: 'hash' + end +end diff --git a/db/migrate/20220406211042_add_deploy_rsync_to_sites.rb b/db/migrate/20220406211042_add_deploy_rsync_to_sites.rb new file mode 100644 index 00000000..92b6f17b --- /dev/null +++ b/db/migrate/20220406211042_add_deploy_rsync_to_sites.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# Agrega un DeployRsync hacia los servidores alternativos para cada +# sitio +class AddDeployRsyncToSites < ActiveRecord::Migration[6.1] + def up + Site.find_each do |site| + SiteService.new(site: site).send :sync_nodes + site.save + end + end + + def down + DeployRsync.destroy_all + end +end diff --git a/db/schema.rb b/db/schema.rb index 107e7be7..a395329d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_05_14_165639) do +ActiveRecord::Schema.define(version: 2021_10_22_225449) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -64,6 +64,7 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do t.string "remote_user" t.boolean "crawler", default: false t.string "http_referer" + t.datetime "created_at", precision: 6 t.index ["geoip2_data_city_name"], name: "index_access_logs_on_geoip2_data_city_name" t.index ["geoip2_data_country_name"], name: "index_access_logs_on_geoip2_data_country_name" t.index ["host"], name: "index_access_logs_on_host" @@ -303,6 +304,15 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do t.index ["usuarie_id"], name: "index_roles_on_usuarie_id" end + create_table "rollups", force: :cascade do |t| + t.string "name", null: false + t.string "interval", null: false + t.datetime "time", null: false + t.jsonb "dimensions", default: {}, null: false + t.float "value" + t.index ["name", "interval", "time", "dimensions"], name: "index_rollups_on_name_and_interval_and_time_and_dimensions", unique: true + end + create_table "sites", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -324,6 +334,15 @@ ActiveRecord::Schema.define(version: 2021_05_14_165639) do t.index ["name"], name: "index_sites_on_name", unique: true end + create_table "stats", force: :cascade do |t| + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.bigint "site_id" + t.string "name", null: false + t.index ["name"], name: "index_stats_on_name", using: :hash + t.index ["site_id"], name: "index_stats_on_site_id" + end + create_table "usuaries", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -370,4 +389,10 @@ new.indexed_content := to_tsvector(('pg_catalog.' || new.dictionary)::regconfig, SQL_ACTIONS end + create_trigger("access_logs_before_insert_row_tr", :compatibility => 1). + on("access_logs"). + before(:insert) do + "new.created_at := to_timestamp(new.msec);" + end + end diff --git a/lib/tasks/stats.rake b/lib/tasks/stats.rake new file mode 100644 index 00000000..9461782a --- /dev/null +++ b/lib/tasks/stats.rake @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +namespace :stats do + desc 'Process stats' + task process_all: :environment do + Site.all.pluck(:id).each do |site_id| + UriCollectionJob.perform_now site_id: site_id, once: true + StatCollectionJob.perform_now site_id: site_id, once: true + end + end +end diff --git a/monit.conf b/monit.conf index 96c08d8a..83d17449 100644 --- a/monit.conf +++ b/monit.conf @@ -25,3 +25,13 @@ check program blazer with path "/usr/local/bin/sutty blazer" every 61 cycles if status != 0 then alert + +check program access_logs + with path "/srv/http/bin/access_logs" as uid "app" and gid "www-data" + every "0 0 * * *" + if status != 0 then alert + +check program stats + with path "/usr/bin/foreman run -f /srv/Procfile -d /srv stats" as uid "rails" gid "www-data" + every "0 1 * * *" + if status != 0 then alert diff --git a/yarn.lock b/yarn.lock index 68b7fd23..50342862 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2257,7 +2257,7 @@ collection-visit@^1.0.0: map-visit "^1.0.0" object-visit "^1.0.0" -color-convert@^1.9.0, color-convert@^1.9.1: +color-convert@^1.9.0, color-convert@^1.9.1, color-convert@^1.9.3: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== @@ -5029,6 +5029,11 @@ mkdirp@^1.0.3, mkdirp@^1.0.4: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== +moment@^2.10.2: + version "2.29.1" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3" + integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ== + move-concurrently@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92"