Sutty/panel
5
0
Fork 0
mirror of https://0xacab.org/sutty/sutty synced 2024-11-16 12:41:41 +00:00
Code Issues Projects Releases Wiki Activity

simplificar el código!!

Browse source 
envía todos los parámetros a la base de datos para acelerar por un
montón la generación de datos
This commit is contained in:
f 2022-04-30 19:35:14 -03:00
parent e8300364d1
commit 26b2d34b4d
3 changed files with 33 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
app/jobs/concerns/recursive_rollup.rb
View file

@ -30,17 +30,16 @@ module RecursiveRollup
# intervalo más amplio. # intervalo más amplio.
# #
# @param :name [String] # @param :name [String]
# @param :beginning [Time]
# @param :operation [Symbol] # @param :operation [Symbol]
# @param :dimensions [Hash] # @param :dimensions [Hash]
# @return [nil] # @return [nil]
def reduce_rollup(name, beginning, operation, **dimensions) def reduce_rollup(name:, dimensions:, operation: :sum)
Stat::INTERVALS.reduce do |previous, current| Stat::INTERVALS.reduce do |previous, current|
recursive_rollup(name: name, recursive_rollup(name: name,
interval_previous: previous, interval_previous: previous,
interval: current, interval: current,
dimensions: dimensions, dimensions: dimensions,
beginning: beginning, beginning: beginning_of_interval,
operation: operation) operation: operation)
# Devolver el intervalo actual # Devolver el intervalo actual

8
app/jobs/stat_collection_job.rb
View file

@ -22,9 +22,11 @@ class StatCollectionJob < PeriodicJob
rollup.average(:seconds) rollup.average(:seconds)
end end
reduce_rollup('builds', beginning, :sum, site_id: site_id) dimensions = { site_id: site_id }
reduce_rollup('space_used', beginning, :average, site_id: site_id)
reduce_rollup('build_time', beginning, :average, site_id: site_id) reduce_rollup(name: 'builds', operation: :sum, dimensions: dimensions)
reduce_rollup(name: 'space_used', operation: :average, dimensions: dimensions)
reduce_rollup(name: 'build_time', operation: :average, dimensions: dimensions)
stat.touch stat.touch
run_again! unless once run_again! unless once

55
app/jobs/uri_collection_job.rb
View file

@ -22,7 +22,7 @@ class UriCollectionJob < PeriodicJob
@site = Site.find site_id @site = Site.find site_id
# Obtener el principio del intervalo anterior # Obtener el principio del intervalo anterior
beginning = beginning_of_interval beginning_of_interval
# Recordar la última vez que se corrió la tarea # Recordar la última vez que se corrió la tarea
stat = site.stats.create! name: STAT_NAME stat = site.stats.create! name: STAT_NAME
# Columnas a agrupar # Columnas a agrupar
@ -33,10 +33,6 @@ class UriCollectionJob < PeriodicJob
# rápido buscar en los rollups indexados que en la tabla en bruto. # rápido buscar en los rollups indexados que en la tabla en bruto.
# #
# Los referers solo se agrupan por host. # Los referers solo se agrupan por host.
site.hostnames.each do |host|
break if stop?
host_dimensions = { host: host }
columns.each_key do |column| columns.each_key do |column|
columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column) columns[column] = AccessLog.where(**host_dimensions).distinct(column).pluck(column)
end end
@ -45,39 +41,25 @@ class UriCollectionJob < PeriodicJob
# que indican las páginas y recursos descargables, el resto son # que indican las páginas y recursos descargables, el resto son
# imágenes, CSS, JS y tipografías que no nos aportan números # imágenes, CSS, JS y tipografías que no nos aportan números
# significativos. # significativos.
name = 'host|uri' uri_dimensions = { host: site.hostnames, uri: uris }
dimensions = { host: host, uri: uris } host_dimensions = { host: site.hostnames }
rollup(name, beginning, **dimensions) # Cantidad de visitas por host
reduce_rollup(name, beginning, :sum, **dimensions) rollup(name: 'host', dimensions: host_dimensions, filter: uri_dimensions)
reduce_rollup(name: 'host', operation: :sum, dimensions: uri_dimensions)
# Cantidad de visitas por página/recurso
rollup(name: 'host|uri', dimensions: uri_dimensions)
reduce_rollup(name: 'host|uri', operation: :sum, dimensions: uri_dimensions)
# Cantidad de visitas host y parámetro
columns.each_pair do |column, values| columns.each_pair do |column, values|
# Obtener orígenes de visitas por host column_name = "host|#{column}"
values.each do |value| column_dimensions = { host: site.hostnames }
column_name = "host|uri|#{column}" column_dimensions[column] = values
column_dimensions = { host: host, uri: uris }
column_dimensions[column] = value
rollup(column_name, beginning, **column_dimensions) rollup(name: column_name, dimensions: column_dimensions, filter: uri_dimensions)
reduce_rollup(column_name, beginning, :sum, **column_dimensions) reduce_rollup(name: column_name, dimensions: column_dimensions)
end
end
# Reducir todas las visitas a cantidad de visitas por host
rollup('host', beginning, **host_dimensions)
# Acumular por mes y año
reduce_rollup('host', beginning, :sum, **host_dimensions)
columns.each_pair do |column, values|
values.each do |value|
column_dimensions = { host: host }
column_dimensions[column] = value
rollup("host|#{column}", beginning, **column_dimensions)
reduce_rollup("host|#{column}", beginning, :sum, **column_dimensions)
end
end
end end
stat.touch stat.touch
@ -92,9 +74,10 @@ class UriCollectionJob < PeriodicJob
# @param :name [String] # @param :name [String]
# @param :beginning [Time] # @param :beginning [Time]
# @param :dimensions [Hash] # @param :dimensions [Hash]
# @param :filter [Hash]
# @return [nil] # @return [nil]
def rollup(name, beginning, **dimensions) def rollup(name:, beginning:, dimensions:, filter: nil)
AccessLog.where(**dimensions) AccessLog.where(**(filter || dimensions))
.where('created_at >= ?', beginning) .where('created_at >= ?', beginning)
.completed_requests .completed_requests
.non_robots .non_robots