From be31823d89e178d3a5e48bdd400c4c566d48c076 Mon Sep 17 00:00:00 2001 From: Martin Edenhofer Date: Wed, 25 Feb 2015 21:52:14 +0100 Subject: [PATCH] Improved cache backend. Heavy reduce of sql queries. --- .gitignore | 4 +- app/models/application_model.rb | 62 +++++++++++++++++++ app/models/email_address.rb | 5 +- app/models/group.rb | 3 +- app/models/organization.rb | 2 +- app/models/role.rb | 3 +- app/models/signature.rb | 3 +- app/models/ticket.rb | 2 + app/models/ticket/article.rb | 4 +- app/models/ticket/state.rb | 2 + app/models/ticket/state_type.rb | 3 +- config/application.rb | 5 +- config/environments/development.rb | 6 +- config/environments/production.rb | 4 ++ config/environments/test.rb | 5 +- lib/cache.rb | 42 +++++++++++++ lib/sessions/backend/collections.rb | 8 +-- lib/sessions/backend/collections/base.rb | 59 ++++-------------- .../backend/collections/organization.rb | 48 -------------- lib/sessions/backend/ticket_create.rb | 12 ++-- lib/sessions/backend/ticket_overview_index.rb | 21 ++++--- lib/sessions/backend/ticket_overview_list.rb | 21 ++++--- lib/sessions/cache_in.rb | 14 +---- test/unit/cache_test.rb | 10 ++- test/unit/ticket_test.rb | 42 +++++++++++++ 25 files changed, 244 insertions(+), 146 deletions(-) diff --git a/.gitignore b/.gitignore index 5fb4727a8..cf824eb70 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,7 @@ # Ignore all logfiles and tempfiles. /log/*.log /tmp/websocket/* -/tmp/cache/* +/tmp/cache* /tmp/pids/* /public/assets/*.* /public/assets/app/* @@ -40,4 +40,4 @@ Gemfile.lock db/schema.rb # Ignore Rubymine config -/.idea \ No newline at end of file +/.idea diff --git a/app/models/application_model.rb b/app/models/application_model.rb index ec512c4a1..fc78118f1 100644 --- a/app/models/application_model.rb +++ b/app/models/application_model.rb @@ -445,6 +445,68 @@ returns =begin +activate latest change on create, update, touch and destroy + +class Model < ApplicationModel + latest_change_support +end + +=end + + def self.latest_change_support + after_create :latest_change_set_from_observer + after_update :latest_change_set_from_observer + after_touch :latest_change_set_from_observer + after_destroy :latest_change_set_from_observer_destroy + end + + def latest_change_set_from_observer + self.class.latest_change_set(self.updated_at) + end + def latest_change_set_from_observer_destroy + self.class.latest_change_set(nil) + end + + def self.latest_change_set(updated_at) + key = "#{self.new.class.name}_latest_change" + expires_in = 31_536_000 # 1 year + + if updated_at == nil + Cache.delete( key ) + else + Cache.write( key, updated_at, { :expires_in => expires_in } ) + end + end + +=begin + + get latest updated_at object timestamp + + latest_change = Ticket.latest_change + +returns + + result = timestamp + +=end + + def self.latest_change + key = "#{self.new.class.name}_latest_change" + updated_at = Cache.get( key ) + + # if we do not have it cached, do lookup + if !updated_at + o = self.select(:updated_at).order(updated_at: :desc).limit(1).first + if o + updated_at = o.updated_at + self.latest_change_set(updated_at) + end + end + updated_at + end + +=begin + activate client notify support on create, update, touch and destroy class Model < ApplicationModel diff --git a/app/models/email_address.rb b/app/models/email_address.rb index fcb01a689..15cb26856 100644 --- a/app/models/email_address.rb +++ b/app/models/email_address.rb @@ -4,4 +4,7 @@ class EmailAddress < ApplicationModel has_many :groups, :after_add => :cache_update, :after_remove => :cache_update validates :realname, :presence => true validates :email, :presence => true -end + + latest_change_support + +end \ No newline at end of file diff --git a/app/models/group.rb b/app/models/group.rb index 2ebb08f5b..3151fa7dd 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -8,4 +8,5 @@ class Group < ApplicationModel activity_stream_support :role => Z_ROLENAME_ADMIN history_support -end + latest_change_support +end \ No newline at end of file diff --git a/app/models/organization.rb b/app/models/organization.rb index a13cdb512..11d7a2e2c 100644 --- a/app/models/organization.rb +++ b/app/models/organization.rb @@ -15,5 +15,5 @@ class Organization < ApplicationModel history_support search_index_support notify_clients_support - + latest_change_support end \ No newline at end of file diff --git a/app/models/role.rb b/app/models/role.rb index 7016fd8ec..58722d401 100644 --- a/app/models/role.rb +++ b/app/models/role.rb @@ -5,4 +5,5 @@ class Role < ApplicationModel has_and_belongs_to_many :users, :after_add => :cache_update, :after_remove => :cache_update validates :name, :presence => true activity_stream_support :role => Z_ROLENAME_ADMIN -end + latest_change_support +end \ No newline at end of file diff --git a/app/models/signature.rb b/app/models/signature.rb index d1cdc6c10..cbeafa0fc 100644 --- a/app/models/signature.rb +++ b/app/models/signature.rb @@ -3,4 +3,5 @@ class Signature < ApplicationModel has_many :groups, :after_add => :cache_update, :after_remove => :cache_update validates :name, :presence => true -end + latest_change_support +end \ No newline at end of file diff --git a/app/models/ticket.rb b/app/models/ticket.rb index e53059863..318ef4fd1 100644 --- a/app/models/ticket.rb +++ b/app/models/ticket.rb @@ -20,6 +20,8 @@ class Ticket < ApplicationModel notify_clients_support + latest_change_support + activity_stream_support :ignore_attributes => { :create_article_type_id => true, :create_article_sender_id => true, diff --git a/app/models/ticket/article.rb b/app/models/ticket/article.rb index 91a1720fb..1f569fde7 100644 --- a/app/models/ticket/article.rb +++ b/app/models/ticket/article.rb @@ -40,9 +40,11 @@ class Ticket::Article < ApplicationModel class Sender < ApplicationModel validates :name, :presence => true + latest_change_support end class Type < ApplicationModel validates :name, :presence => true + latest_change_support end -end +end \ No newline at end of file diff --git a/app/models/ticket/state.rb b/app/models/ticket/state.rb index a4a486366..8987a994f 100644 --- a/app/models/ticket/state.rb +++ b/app/models/ticket/state.rb @@ -4,6 +4,8 @@ class Ticket::State < ApplicationModel belongs_to :state_type, :class_name => 'Ticket::StateType' validates :name, :presence => true + latest_change_support + =begin list tickets by customer diff --git a/app/models/ticket/state_type.rb b/app/models/ticket/state_type.rb index 178046c95..162b014e1 100644 --- a/app/models/ticket/state_type.rb +++ b/app/models/ticket/state_type.rb @@ -3,4 +3,5 @@ class Ticket::StateType < ApplicationModel has_many :states, :class_name => 'Ticket::State' validates :name, :presence => true -end + latest_change_support +end \ No newline at end of file diff --git a/config/application.rb b/config/application.rb index a30b02dc5..194120a96 100644 --- a/config/application.rb +++ b/config/application.rb @@ -66,11 +66,8 @@ module Zammad # Version of your assets, change this if you want to expire all your assets config.assets.version = '1.0' - # Use a different cache store in production - config.cache_store = :file_store, 'tmp/cache/file_store' - # REST api path config.api_path = '/api/v1' end -end +end \ No newline at end of file diff --git a/config/environments/development.rb b/config/environments/development.rb index 9a68eac06..e1f3b84eb 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -37,4 +37,8 @@ Zammad::Application.configure do :live_reload_port => 35738, :source => :vendored ) -end + + # define cache store + config.cache_store = :file_store, 'tmp/cache_file_store_development' + +end \ No newline at end of file diff --git a/config/environments/production.rb b/config/environments/production.rb index a07e81ebd..98ce96c7c 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -77,4 +77,8 @@ Zammad::Application.configure do # Use default logging formatter so that PID and timestamp are not suppressed config.log_formatter = ::Logger::Formatter.new + + # define cache store + config.cache_store = :file_store, 'tmp/cache_file_store_production' + end diff --git a/config/environments/test.rb b/config/environments/test.rb index 8c8d17be4..64e96eec1 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -46,4 +46,7 @@ Zammad::Application.configure do # Enable autoload config.dependency_loading = true -end + # define cache store + config.cache_store = :file_store, 'tmp/cache_file_store_test' + +end \ No newline at end of file diff --git a/lib/cache.rb b/lib/cache.rb index c7e3a37d9..1fb5f590f 100644 --- a/lib/cache.rb +++ b/lib/cache.rb @@ -1,8 +1,32 @@ module Cache + +=begin + +delete a cache + + Cache.delete( 'some_key' ) + +=end + def self.delete( key ) # puts 'Cache.delete' + key.to_s Rails.cache.delete( key.to_s ) end + +=begin + +write a cache + + Cache.write( + 'some_key', + { :some => { :data => { 'structure' } } }, + { + :expires_in => 24.hours, # optional + } + ) + +=end + def self.write( key, data, params = {} ) if !params[:expires_in] params[:expires_in] = 24.hours @@ -14,10 +38,28 @@ module Cache puts "NOTICE: #{e.message}" end end + +=begin + +get a cache + + value = Cache.write( 'some_key' ) + +=end + def self.get( key ) # puts 'Cache.get: ' + key.to_s Rails.cache.read( key.to_s ) end + +=begin + +clear whole cache store + + Cache.clear + +=end + def self.clear # puts 'Cache.clear...' # workaround, set test cache before clear whole cache, Rails.cache.clear complains about not existing cache dir diff --git a/lib/sessions/backend/collections.rb b/lib/sessions/backend/collections.rb index 95b49ee85..fdb176d5f 100644 --- a/lib/sessions/backend/collections.rb +++ b/lib/sessions/backend/collections.rb @@ -1,10 +1,10 @@ class Sessions::Backend::Collections def initialize( user, client, client_id ) - @user = user - @client = client - @client_id = client_id - @backends = self.backend + @user = user + @client = client + @client_id = client_id + @backends = self.backend end diff --git a/lib/sessions/backend/collections/base.rb b/lib/sessions/backend/collections/base.rb index 7f5ba83f7..2c187c442 100644 --- a/lib/sessions/backend/collections/base.rb +++ b/lib/sessions/backend/collections/base.rb @@ -2,56 +2,16 @@ class Sessions::Backend::Collections::Base class << self; attr_accessor :model, :is_role, :is_not_role end def initialize( user, client = nil, client_id = nil ) - @user = user - @client = client - @client_id = client_id - @last_change = nil - end - - def collection_key - "collections::load::#{ self.class.to_s }::#{ @user.id }" + @user = user + @client = client + @client_id = client_id + @last_change = nil end def load -#puts "-LOAD--------#{self.collection_key}" - # check timeout - cache = Sessions::CacheIn.get( self.collection_key ) - return cache if @last_change && cache -#puts "---REAL FETCH #{@user.id}" - # update last changed - last = self.class.model.constantize.select('updated_at').order('updated_at DESC').first - if last - @last_change = last.updated_at - end - - # if no entry exists, remember last check - if !@last_change - @last_change = Time.now - end # get whole collection - all = self.class.model.constantize.all - - # set new timeout - Sessions::CacheIn.set( self.collection_key, all, { :expires_in => 10.minutes } ) - - all - end - - def changed? - # if no data has been delivered till now - return true if !@last_change - - # check if update has been done - last = self.class.model.constantize.select('updated_at').order('updated_at DESC').first - return false if !last - return false if last.updated_at == @last_change - - # delete collection cache - Sessions::CacheIn.delete( self.collection_key ) - - # collection has changed - true + self.class.model.constantize.all end def client_key @@ -83,13 +43,18 @@ class Sessions::Backend::Collections::Base # set new timeout Sessions::CacheIn.set( self.client_key, true, { :expires_in => 10.seconds } ) - return if !self.changed? + # check if update has been done + last_change = self.class.model.constantize.latest_change + return if last_change == @last_change + @last_change = last_change + + # load current data items = self.load return if !items||items.empty? # get relations of data - all = [] + all = [] items.each {|item| all.push item.attributes_with_associations } diff --git a/lib/sessions/backend/collections/organization.rb b/lib/sessions/backend/collections/organization.rb index f3d1dbd97..7d9666059 100644 --- a/lib/sessions/backend/collections/organization.rb +++ b/lib/sessions/backend/collections/organization.rb @@ -3,28 +3,6 @@ class Sessions::Backend::Collections::Organization < Sessions::Backend::Collecti def load - # check timeout - cache = Sessions::CacheIn.get( self.collection_key ) - return cache if @last_change && cache - - # update last changed - if !@user.is_role('Customer') - last = self.class.model.constantize.select('updated_at').order('updated_at DESC').first - if last - @last_change = last.updated_at - end - else - if @user.organization_id - last = Organization.where( :id => @user.organization_id ).first - @last_change = last.updated_at - end - end - - # if no entry exists, remember last check - if !@last_change - @last_change = Time.now - end - # get whole collection all = [] if !@user.is_role('Customer') @@ -35,33 +13,7 @@ class Sessions::Backend::Collections::Organization < Sessions::Backend::Collecti end end - # set new timeout - Sessions::CacheIn.set( self.collection_key, all, { :expires_in => 10.minutes } ) - all end - def changed? - - # if no data has been delivered till now - return true if !@last_change - - # check if update has been done - if !@user.is_role('Customer') - last = self.class.model.constantize.select('updated_at').order('updated_at DESC').first - else - if @user.organization_id - last = Organization.where( :id => @user.organization_id ).first - end - end - return false if !last - return false if last.updated_at == @last_change - - # delete collection cache - Sessions::CacheIn.delete( self.collection_key ) - - # collection has changed - true - end - end \ No newline at end of file diff --git a/lib/sessions/backend/ticket_create.rb b/lib/sessions/backend/ticket_create.rb index f5bf8bb14..ac1ad249e 100644 --- a/lib/sessions/backend/ticket_create.rb +++ b/lib/sessions/backend/ticket_create.rb @@ -1,16 +1,16 @@ class Sessions::Backend::TicketCreate def initialize( user, client = nil, client_id = nil ) - @user = user - @client = client - @client_id = client_id - @last_change = nil + @user = user + @client = client + @client_id = client_id + @last_change = nil end def load # get attributes to update ticket_create_attributes = Ticket::ScreenOptions.attributes_to_change( - :user => @user.id, + :user => @user.id, ) # no data exists @@ -36,7 +36,7 @@ class Sessions::Backend::TicketCreate return if timeout # set new timeout - Sessions::CacheIn.set( self.client_key, true, { :expires_in => 30.seconds } ) + Sessions::CacheIn.set( self.client_key, true, { :expires_in => 60.seconds } ) ticket_create_attributes = self.load diff --git a/lib/sessions/backend/ticket_overview_index.rb b/lib/sessions/backend/ticket_overview_index.rb index dc7cdf87e..ca81bf326 100644 --- a/lib/sessions/backend/ticket_overview_index.rb +++ b/lib/sessions/backend/ticket_overview_index.rb @@ -1,9 +1,10 @@ class Sessions::Backend::TicketOverviewIndex def initialize( user, client = nil, client_id = nil ) - @user = user - @client = client - @client_id = client_id - @last_change = nil + @user = user + @client = client + @client_id = client_id + @last_change = nil + @last_ticket_change = nil end def load @@ -31,13 +32,19 @@ class Sessions::Backend::TicketOverviewIndex def push - # check timeout + # check check interval timeout = Sessions::CacheIn.get( self.client_key ) return if timeout - # set new timeout - Sessions::CacheIn.set( self.client_key, true, { :expires_in => 20.seconds } ) + # reset check interval + Sessions::CacheIn.set( self.client_key, true, { :expires_in => 5.seconds } ) + # check if min one ticket has changed + last_ticket_change = Ticket.latest_change + return if last_ticket_change == @last_ticket_change + @last_ticket_change = last_ticket_change + + # load current data data = self.load return if !data diff --git a/lib/sessions/backend/ticket_overview_list.rb b/lib/sessions/backend/ticket_overview_list.rb index bb86d6d1b..f0d8546b0 100644 --- a/lib/sessions/backend/ticket_overview_list.rb +++ b/lib/sessions/backend/ticket_overview_list.rb @@ -1,9 +1,10 @@ class Sessions::Backend::TicketOverviewList def initialize( user, client = nil, client_id = nil ) - @user = user - @client = client - @client_id = client_id - @last_change = nil + @user = user + @client = client + @client_id = client_id + @last_change = nil + @last_ticket_change = nil end def load @@ -41,13 +42,19 @@ class Sessions::Backend::TicketOverviewList def push - # check timeout + # check interval timeout = Sessions::CacheIn.get( self.client_key ) return if timeout - # set new timeout - Sessions::CacheIn.set( self.client_key, true, { :expires_in => 20.seconds } ) + # reset check interval + Sessions::CacheIn.set( self.client_key, true, { :expires_in => 6.seconds } ) + # check if min one ticket has changed + last_ticket_change = Ticket.latest_change + return if last_ticket_change == @last_ticket_change + @last_ticket_change = last_ticket_change + + # load current data items = self.load return if !items diff --git a/lib/sessions/cache_in.rb b/lib/sessions/cache_in.rb index e0b6dcc61..d25fbd720 100644 --- a/lib/sessions/cache_in.rb +++ b/lib/sessions/cache_in.rb @@ -12,10 +12,10 @@ module Sessions::CacheIn def self.set( key, value, params = {} ) # puts 'CacheIn.set:' + key + '-' + value.inspect if params[:expires_in] - @@expires_in[key] = Time.now + params[:expires_in] + @@expires_in[key] = Time.now + params[:expires_in] @@expires_in_ttl[key] = params[:expires_in] end - @@data[ key ] = value + @@data[ key ] = value @@data_time[ key ] = Time.now end @@ -45,17 +45,9 @@ module Sessions::CacheIn false end - def self.get_time( key, params = {} ) - data = self.get( key, params ) - if data - return @@data_time[key] - end - nil - end - def self.get( key, params = {} ) # puts 'CacheIn.get:' + key + '-' + @@data[ key ].inspect return if self.expired( key, params ) @@data[ key ] end -end +end \ No newline at end of file diff --git a/test/unit/cache_test.rb b/test/unit/cache_test.rb index b397e08e6..1af177008 100644 --- a/test/unit/cache_test.rb +++ b/test/unit/cache_test.rb @@ -1,6 +1,6 @@ # encoding: utf-8 require 'test_helper' - + class CacheTest < ActiveSupport::TestCase test 'cache' do tests = [ @@ -110,4 +110,12 @@ class CacheTest < ActiveSupport::TestCase end } end + + # verify if second cache write overwrite first one + test 'cache reset' do + Cache.write( 'some_reset_key', 123 ) + Cache.write( 'some_reset_key', 12356 ) + cache = Cache.get( 'some_reset_key' ) + assert_equal( cache, 12356, 'verify' ) + end end \ No newline at end of file diff --git a/test/unit/ticket_test.rb b/test/unit/ticket_test.rb index efb0e5da1..384e86d6d 100644 --- a/test/unit/ticket_test.rb +++ b/test/unit/ticket_test.rb @@ -123,4 +123,46 @@ class TicketTest < ActiveSupport::TestCase delete = ticket.destroy assert( delete, "ticket destroy" ) end + + + test 'ticket latest change' do + ticket1 = Ticket.create( + :title => 'latest change 1', + :group => Group.lookup( :name => 'Users'), + :customer_id => 2, + :state => Ticket::State.lookup( :name => 'new' ), + :priority => Ticket::Priority.lookup( :name => '2 normal' ), + :updated_by_id => 1, + :created_by_id => 1, + ) + assert_equal( Ticket.latest_change.to_s, ticket1.updated_at.to_s ) + + sleep 1 + + ticket2 = Ticket.create( + :title => 'latest change 2', + :group => Group.lookup( :name => 'Users'), + :customer_id => 2, + :state => Ticket::State.lookup( :name => 'new' ), + :priority => Ticket::Priority.lookup( :name => '2 normal' ), + :updated_by_id => 1, + :created_by_id => 1, + ) + assert_equal( Ticket.latest_change.to_s, ticket2.updated_at.to_s ) + + sleep 1 + + ticket1.title = 'latest change 1 - 1' + ticket1.save + assert_equal( Ticket.latest_change.to_s, ticket1.updated_at.to_s ) + + sleep 1 + + ticket1.touch + assert_equal( Ticket.latest_change.to_s, ticket1.updated_at.to_s ) + + ticket1.destroy + assert_equal( Ticket.latest_change.to_s, ticket2.updated_at.to_s ) + + end end \ No newline at end of file