Improved cache backend. Heavy reduce of sql queries.

.gitignore

@@ -13,7 +13,7 @@
 # Ignore all logfiles and tempfiles.
 /log/*.log
 /tmp/websocket/*
-/tmp/cache/*
+/tmp/cache*
 /tmp/pids/*
 /public/assets/*.*
 /public/assets/app/*
@@ -40,4 +40,4 @@ Gemfile.lock
 db/schema.rb
 
 # Ignore Rubymine config
-/.idea
+/.idea

app/models/application_model.rb

@@ -445,6 +445,68 @@ returns
 
 =begin
 
+activate latest change on create, update, touch and destroy
+
+class Model < ApplicationModel
+  latest_change_support
+end
+
+=end
+
+  def self.latest_change_support
+    after_create  :latest_change_set_from_observer
+    after_update  :latest_change_set_from_observer
+    after_touch   :latest_change_set_from_observer
+    after_destroy :latest_change_set_from_observer_destroy
+  end
+
+  def latest_change_set_from_observer
+    self.class.latest_change_set(self.updated_at)
+  end
+  def latest_change_set_from_observer_destroy
+    self.class.latest_change_set(nil)
+  end
+
+  def self.latest_change_set(updated_at)
+    key        = "#{self.new.class.name}_latest_change"
+    expires_in = 31_536_000 # 1 year
+
+    if updated_at == nil
+      Cache.delete( key )
+    else
+      Cache.write( key, updated_at, { :expires_in => expires_in } )
+    end
+  end
+
+=begin
+
+  get latest updated_at object timestamp
+
+  latest_change = Ticket.latest_change
+
+returns
+
+  result = timestamp
+
+=end
+
+  def self.latest_change
+    key        = "#{self.new.class.name}_latest_change"
+    updated_at = Cache.get( key )
+
+    # if we do not have it cached, do lookup
+    if !updated_at
+      o = self.select(:updated_at).order(updated_at: :desc).limit(1).first
+      if o
+        updated_at = o.updated_at
+        self.latest_change_set(updated_at)
+      end
+    end
+    updated_at
+  end
+
+=begin
+
 activate client notify support on create, update, touch and destroy
 
 class Model < ApplicationModel

app/models/email_address.rb

@@ -4,4 +4,7 @@ class EmailAddress < ApplicationModel
   has_many                :groups,   :after_add => :cache_update, :after_remove => :cache_update
   validates               :realname, :presence => true
   validates               :email,    :presence => true
-end
+
+  latest_change_support
+
+end

app/models/group.rb

@@ -8,4 +8,5 @@ class Group < ApplicationModel
 
   activity_stream_support  :role => Z_ROLENAME_ADMIN
   history_support
-end
+  latest_change_support
+end

app/models/organization.rb

@@ -15,5 +15,5 @@ class Organization < ApplicationModel
   history_support
   search_index_support
   notify_clients_support
-
+  latest_change_support
 end

app/models/role.rb

@@ -5,4 +5,5 @@ class Role < ApplicationModel
   has_and_belongs_to_many   :users, :after_add => :cache_update, :after_remove => :cache_update
   validates                 :name,  :presence => true
   activity_stream_support   :role => Z_ROLENAME_ADMIN
-end
+  latest_change_support
+end

app/models/signature.rb

@@ -3,4 +3,5 @@
 class Signature < ApplicationModel
   has_many                :groups,  :after_add => :cache_update, :after_remove => :cache_update
   validates               :name,    :presence => true
-end
+  latest_change_support
+end

app/models/ticket.rb

@@ -20,6 +20,8 @@ class Ticket < ApplicationModel
 
   notify_clients_support
 
+  latest_change_support
+
   activity_stream_support :ignore_attributes => {
     :create_article_type_id   => true,
     :create_article_sender_id => true,

app/models/ticket/article.rb

@@ -40,9 +40,11 @@ class Ticket::Article < ApplicationModel
 
   class Sender < ApplicationModel
     validates   :name, :presence => true
+    latest_change_support
   end
 
   class Type < ApplicationModel
     validates   :name, :presence => true
+    latest_change_support
   end
-end
+end

app/models/ticket/state.rb

@@ -4,6 +4,8 @@ class Ticket::State < ApplicationModel
   belongs_to    :state_type,        :class_name => 'Ticket::StateType'
   validates     :name, :presence => true
 
+  latest_change_support
+
 =begin
 
 list tickets by customer

app/models/ticket/state_type.rb

@@ -3,4 +3,5 @@
 class Ticket::StateType < ApplicationModel
   has_many      :states,            :class_name => 'Ticket::State'
   validates     :name, :presence => true
-end
+  latest_change_support
+end

config/application.rb

@@ -66,11 +66,8 @@ module Zammad
     # Version of your assets, change this if you want to expire all your assets
     config.assets.version = '1.0'
 
-    # Use a different cache store in production
-    config.cache_store = :file_store, 'tmp/cache/file_store'
-
     # REST api path
     config.api_path = '/api/v1'
 
   end
-end
+end

config/environments/development.rb

@@ -37,4 +37,8 @@ Zammad::Application.configure do
     :live_reload_port => 35738,
     :source => :vendored
   )
-end
+
+  # define cache store
+  config.cache_store = :file_store, 'tmp/cache_file_store_development'
+
+end

config/environments/production.rb

@@ -77,4 +77,8 @@ Zammad::Application.configure do
 
   # Use default logging formatter so that PID and timestamp are not suppressed
   config.log_formatter = ::Logger::Formatter.new
+
+  # define cache store
+  config.cache_store = :file_store, 'tmp/cache_file_store_production'
+
 end

config/environments/test.rb

@@ -46,4 +46,7 @@ Zammad::Application.configure do
   # Enable autoload
   config.dependency_loading = true
 
-end
+  # define cache store
+  config.cache_store = :file_store, 'tmp/cache_file_store_test'
+
+end

lib/cache.rb

@@ -1,8 +1,32 @@
 module Cache
+
+=begin
+
+delete a cache
+
+  Cache.delete( 'some_key' )
+
+=end
+
   def self.delete( key )
 #    puts 'Cache.delete' + key.to_s
     Rails.cache.delete( key.to_s )
   end
+
+=begin
+
+write a cache
+
+  Cache.write(
+    'some_key',
+    { :some => { :data => { 'structure' } } },
+    {
+      :expires_in => 24.hours, # optional
+    }
+  )
+
+=end
+
   def self.write( key, data, params = {} )
     if !params[:expires_in]
       params[:expires_in] = 24.hours
@@ -14,10 +38,28 @@ module Cache
       puts "NOTICE: #{e.message}"
     end
   end
+
+=begin
+
+get a cache
+
+  value = Cache.write( 'some_key' )
+
+=end
+
   def self.get( key )
 #    puts 'Cache.get: ' + key.to_s
     Rails.cache.read( key.to_s )
   end
+
+=begin
+
+clear whole cache store
+
+  Cache.clear
+
+=end
+
   def self.clear
 #    puts 'Cache.clear...'
     # workaround, set test cache before clear whole cache, Rails.cache.clear complains about not existing cache dir

lib/sessions/backend/collections.rb

@@ -1,10 +1,10 @@
 class Sessions::Backend::Collections
 
   def initialize( user, client, client_id )
-    @user       = user
+    @user      = user
-    @client     = client
+    @client    = client
-    @client_id  = client_id
+    @client_id = client_id
-    @backends   = self.backend
+    @backends  = self.backend
   end
 
 

lib/sessions/backend/collections/base.rb

@@ -2,56 +2,16 @@ class Sessions::Backend::Collections::Base
   class << self; attr_accessor :model, :is_role, :is_not_role end
 
   def initialize( user, client = nil, client_id = nil )
-    @user         = user
+    @user        = user
-    @client       = client
+    @client      = client
-    @client_id    = client_id
+    @client_id   = client_id
-    @last_change  = nil
+    @last_change = nil
-  end
-
-  def collection_key
-    "collections::load::#{ self.class.to_s }::#{ @user.id }"
   end
 
   def load
-#puts "-LOAD--------#{self.collection_key}"
-    # check timeout
-    cache = Sessions::CacheIn.get( self.collection_key )
-    return cache if @last_change && cache
-#puts "---REAL FETCH #{@user.id}"
-    # update last changed
-    last = self.class.model.constantize.select('updated_at').order('updated_at DESC').first
-    if last
-      @last_change = last.updated_at
-    end
-
-    # if no entry exists, remember last check
-    if !@last_change
-      @last_change = Time.now
-    end
 
     # get whole collection
-    all = self.class.model.constantize.all
+    self.class.model.constantize.all
-
-    # set new timeout
-    Sessions::CacheIn.set( self.collection_key, all, { :expires_in => 10.minutes } )
-
-    all
-  end
-
-  def changed?
-    # if no data has been delivered till now
-    return true if !@last_change
-
-    # check if update has been done
-    last = self.class.model.constantize.select('updated_at').order('updated_at DESC').first
-    return false if !last
-    return false if last.updated_at == @last_change
-
-    # delete collection cache
-    Sessions::CacheIn.delete( self.collection_key )
-
-    # collection has changed
-    true
   end
 
   def client_key
@@ -83,13 +43,18 @@ class Sessions::Backend::Collections::Base
     # set new timeout
     Sessions::CacheIn.set( self.client_key, true, { :expires_in => 10.seconds } )
 
-    return if !self.changed?
+    # check if update has been done
+    last_change = self.class.model.constantize.latest_change
+    return if last_change == @last_change
+    @last_change = last_change
+
+    # load current data
     items = self.load
 
     return if !items||items.empty?
 
     # get relations of data
-    all   = []
+    all = []
     items.each {|item|
       all.push item.attributes_with_associations
     }

lib/sessions/backend/collections/organization.rb

@@ -3,28 +3,6 @@ class Sessions::Backend::Collections::Organization < Sessions::Backend::Collecti
 
   def load
 
-    # check timeout
-    cache = Sessions::CacheIn.get( self.collection_key )
-    return cache if @last_change && cache
-
-    # update last changed
-    if !@user.is_role('Customer')
-      last = self.class.model.constantize.select('updated_at').order('updated_at DESC').first
-      if last
-        @last_change = last.updated_at
-      end
-    else
-      if @user.organization_id
-        last = Organization.where( :id => @user.organization_id ).first
-        @last_change = last.updated_at
-      end
-    end
-
-    # if no entry exists, remember last check
-    if !@last_change
-      @last_change = Time.now
-    end
-
     # get whole collection
     all = []
     if !@user.is_role('Customer')
@@ -35,33 +13,7 @@ class Sessions::Backend::Collections::Organization < Sessions::Backend::Collecti
       end
     end
 
-    # set new timeout
-    Sessions::CacheIn.set( self.collection_key, all, { :expires_in => 10.minutes } )
-
     all
   end
 
-  def changed?
-
-    # if no data has been delivered till now
-    return true if !@last_change
-
-    # check if update has been done
-    if !@user.is_role('Customer')
-      last = self.class.model.constantize.select('updated_at').order('updated_at DESC').first
-    else
-      if @user.organization_id
-        last = Organization.where( :id => @user.organization_id ).first
-      end
-    end
-    return false if !last
-    return false if last.updated_at == @last_change
-
-    # delete collection cache
-    Sessions::CacheIn.delete( self.collection_key )
-
-    # collection has changed
-    true
-  end
-
 end

lib/sessions/backend/ticket_create.rb

@@ -1,16 +1,16 @@
 class Sessions::Backend::TicketCreate
   def initialize( user, client = nil, client_id = nil )
-    @user         = user
+    @user        = user
-    @client       = client
+    @client      = client
-    @client_id    = client_id
+    @client_id   = client_id
-    @last_change  = nil
+    @last_change = nil
   end
 
   def load
 
     # get attributes to update
     ticket_create_attributes = Ticket::ScreenOptions.attributes_to_change(
-      :user   => @user.id,
+      :user => @user.id,
     )
 
     # no data exists
@@ -36,7 +36,7 @@ class Sessions::Backend::TicketCreate
     return if timeout
 
     # set new timeout
-    Sessions::CacheIn.set( self.client_key, true, { :expires_in => 30.seconds } )
+    Sessions::CacheIn.set( self.client_key, true, { :expires_in => 60.seconds } )
 
     ticket_create_attributes = self.load
 

lib/sessions/backend/ticket_overview_index.rb

@@ -1,9 +1,10 @@
 class Sessions::Backend::TicketOverviewIndex
   def initialize( user, client = nil, client_id = nil )
-    @user         = user
+    @user               = user
-    @client       = client
+    @client             = client
-    @client_id    = client_id
+    @client_id          = client_id
-    @last_change  = nil
+    @last_change        = nil
+    @last_ticket_change = nil
   end
 
   def load
@@ -31,13 +32,19 @@ class Sessions::Backend::TicketOverviewIndex
 
   def push
 
-    # check timeout
+    # check check interval
     timeout = Sessions::CacheIn.get( self.client_key )
     return if timeout
 
-    # set new timeout
+    # reset check interval
-    Sessions::CacheIn.set( self.client_key, true, { :expires_in => 20.seconds } )
+    Sessions::CacheIn.set( self.client_key, true, { :expires_in => 5.seconds } )
 
+    # check if min one ticket has changed
+    last_ticket_change = Ticket.latest_change
+    return if last_ticket_change == @last_ticket_change
+    @last_ticket_change = last_ticket_change
+
+    # load current data
     data = self.load
 
     return if !data

lib/sessions/backend/ticket_overview_list.rb

@@ -1,9 +1,10 @@
 class Sessions::Backend::TicketOverviewList
   def initialize( user, client = nil, client_id = nil )
-    @user         = user
+    @user               = user
-    @client       = client
+    @client             = client
-    @client_id    = client_id
+    @client_id          = client_id
-    @last_change  = nil
+    @last_change        = nil
+    @last_ticket_change = nil
   end
 
   def load
@@ -41,13 +42,19 @@ class Sessions::Backend::TicketOverviewList
 
   def push
 
-    # check timeout
+    # check interval
     timeout = Sessions::CacheIn.get( self.client_key )
     return if timeout
 
-    # set new timeout
+    # reset check interval
-    Sessions::CacheIn.set( self.client_key, true, { :expires_in => 20.seconds } )
+    Sessions::CacheIn.set( self.client_key, true, { :expires_in => 6.seconds } )
 
+    # check if min one ticket has changed
+    last_ticket_change = Ticket.latest_change
+    return if last_ticket_change == @last_ticket_change
+    @last_ticket_change = last_ticket_change
+
+    # load current data
     items = self.load
 
     return if !items

lib/sessions/cache_in.rb

@@ -12,10 +12,10 @@ module Sessions::CacheIn
   def self.set( key, value, params = {} )
 #    puts 'CacheIn.set:' + key + '-' + value.inspect
     if params[:expires_in]
-      @@expires_in[key] = Time.now + params[:expires_in]
+      @@expires_in[key]     = Time.now + params[:expires_in]
       @@expires_in_ttl[key] = params[:expires_in]
     end
-    @@data[ key ] = value
+    @@data[ key ]      = value
     @@data_time[ key ] = Time.now
   end
 
@@ -45,17 +45,9 @@ module Sessions::CacheIn
     false
   end
 
-  def self.get_time( key, params = {} )
-    data = self.get( key, params )
-    if data
-      return @@data_time[key]
-    end
-    nil
-  end
-
   def self.get( key, params = {} )
 #    puts 'CacheIn.get:' + key + '-' + @@data[ key ].inspect
     return if self.expired( key, params )
     @@data[ key ]
   end
-end
+end

test/unit/cache_test.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 require 'test_helper'
- 
+
 class CacheTest < ActiveSupport::TestCase
   test 'cache' do
     tests = [
@@ -110,4 +110,12 @@ class CacheTest < ActiveSupport::TestCase
       end
     }
   end
+
+  # verify if second cache write overwrite first one
+  test 'cache reset' do
+    Cache.write( 'some_reset_key', 123 )
+    Cache.write( 'some_reset_key', 12356 )
+    cache = Cache.get( 'some_reset_key' )
+    assert_equal( cache, 12356, 'verify' )
+  end
 end

test/unit/ticket_test.rb

@@ -123,4 +123,46 @@ class TicketTest < ActiveSupport::TestCase
     delete = ticket.destroy
     assert( delete, "ticket destroy" )
   end
+
+
+  test 'ticket latest change' do
+    ticket1 = Ticket.create(
+      :title         => 'latest change 1',
+      :group         => Group.lookup( :name => 'Users'),
+      :customer_id   => 2,
+      :state         => Ticket::State.lookup( :name => 'new' ),
+      :priority      => Ticket::Priority.lookup( :name => '2 normal' ),
+      :updated_by_id => 1,
+      :created_by_id => 1,
+    )
+    assert_equal( Ticket.latest_change.to_s, ticket1.updated_at.to_s )
+
+    sleep 1
+
+    ticket2 = Ticket.create(
+      :title         => 'latest change 2',
+      :group         => Group.lookup( :name => 'Users'),
+      :customer_id   => 2,
+      :state         => Ticket::State.lookup( :name => 'new' ),
+      :priority      => Ticket::Priority.lookup( :name => '2 normal' ),
+      :updated_by_id => 1,
+      :created_by_id => 1,
+    )
+    assert_equal( Ticket.latest_change.to_s, ticket2.updated_at.to_s )
+
+    sleep 1
+
+    ticket1.title = 'latest change 1 - 1'
+    ticket1.save
+    assert_equal( Ticket.latest_change.to_s, ticket1.updated_at.to_s )
+
+    sleep 1
+
+    ticket1.touch
+    assert_equal( Ticket.latest_change.to_s, ticket1.updated_at.to_s )
+
+    ticket1.destroy
+    assert_equal( Ticket.latest_change.to_s, ticket2.updated_at.to_s )
+
+  end
 end