Moved to dedicated provider backend modules.

app/models/store.rb

@@ -2,7 +2,11 @@
 
 require 'digest/md5'
 
+
 class Store < ApplicationModel
+  require 'store/object'
+  require 'store/file'
+
   store       :preferences
   belongs_to  :store_object,          :class_name => 'Store::Object'
   belongs_to  :store_file,            :class_name => 'Store::File'
@@ -35,28 +39,10 @@ returns
     store_object = Store::Object.create_if_not_exists( :name => data['object'] )
     data['store_object_id'] = store_object.id
 
-    # check if record already exists
-    #    store = Store.where( :store_object_id => store_object.id, :o_id => data['o_id'],  ).first
-    #    if store != nil
-    #      return store
-    #    end
+    # add to real store
+    file = Store::File.add( data['data'] )
 
-    # check real store
-    md5 = Digest::MD5.hexdigest( data['data'] )
     data['size'] = data['data'].to_s.bytesize
-
-    # file = Store::Provider::DB.create( data['data'], md5 )
-
-    file = Store::File.where( :md5 => md5 ).first
-
-    # store attachment
-    if file == nil
-      file = Store::File.create(
-        :data => data['data'],
-        :md5  => md5,
-      )
-    end
-
     data['store_file_id'] = file.id
 
     # not needed attributes
@@ -128,7 +114,6 @@ returns
       # check backend for references
       files = Store.where( :store_file_id => store.store_file_id )
       if files.count == 1 && files.first.id == store.id
-        # file = Store::Provider::DB.delete( store.store_file_id )
         Store::File.find( store.store_file_id ).destroy
       end
 
@@ -137,165 +122,19 @@ returns
     return true
   end
 
-  # get attachment
   def content
-    # Store::Provider::DB.content( store.store_file_id )
     file = Store::File.where( :id => self.store_file_id ).first
-    return if !file
-    if file.file_system
-      return file.read_from_fs
+    if !file
+      raise "No such file #{ self.store_file_id }!"
     end
-    file.data
+    file.content
   end
-end
 
-class Store::Object < ApplicationModel
-  validates :name, :presence => true
-end
-
-class Store::File < ApplicationModel
-  before_validation :add_md5
-  before_create     :check_location
-  after_destroy     :unlink_location
-
-  # generate file location
-  def get_locaton
-
-    # generate directory
-    base = Rails.root.to_s + '/storage/fs/'
-    parts = self.md5.scan(/.{1,3}/)
-    path = parts[ 1 .. 7 ].join('/') + '/'
-    file = parts[ 8 .. parts.count ].join('')
-    location = "#{base}/#{path}"
-
-    # create directory if not exists
-    if !File.exist?( location )
-      FileUtils.mkdir_p( location )
-    end
-    location += file
-  end
-
-  # read file from fs
-  def unlink_location
-    if File.exist?( self.get_locaton )
-      puts "NOTICE: storge remove '#{self.get_locaton}'"
-      File.delete( self.get_locaton )
-    end
-  end
-
-  # read file from fs
-  def read_from_fs
-    puts "read from fs #{self.get_locaton}"
-    return if !File.exist?( self.get_locaton )
-    data = File.open( self.get_locaton, 'rb' )
-    content = data.read
-
-    # check md5
-    md5 = Digest::MD5.hexdigest( content )
-    if md5 != self.md5
-      raise "ERROR: Corrupt file in fs #{self.get_locaton}, md5 should be #{self.md5} but is #{md5}"
-    end
-    content
-  end
-
-  # write file to fs
-  def write_to_fs
-
-    # install file
-    permission = '600'
-    if !File.exist?( self.get_locaton )
-      puts "NOTICE: storge write '#{self.get_locaton}' (#{permission})"
-      file = File.new( self.get_locaton, 'wb' )
-      file.write( self.data )
-      file.close
-    end
-    File.chmod( permission.to_i(8), self.get_locaton )
-
-    # check md5
-    md5 = Digest::MD5.hexdigest( self.read_from_fs )
-    if md5 != self.md5
-      raise "ERROR: Corrupt file in fs #{self.get_locaton}, md5 should be #{self.md5} but is #{md5}"
-    end
-
-    true
-  end
-
-  # write file to db
-  def write_to_db
-
-    # read and check md5
-    content = self.read_from_fs
-
-    # store in database
-    self.data = content
-    self.save
-
-    # check md5 against db content
-    md5 = Digest::MD5.hexdigest( self.data )
-    if md5 != self.md5
-      raise "ERROR: Corrupt file in db #{self.get_locaton}, md5 should be #{self.md5} but is #{md5}"
-    end
-
-    true
-  end
-
-  # check database data and md5, in case fix it
-  def self.db_check_md5(fix_it = nil)
-    Store::File.where( :file_system => false ).each {|item|
-      md5 = Digest::MD5.hexdigest( item.data )
-      if md5 != item.md5
-        puts "DIFF: md5 diff of Store::File.find(#{item.id}) "
-        if fix_it
-          item.update_attribute( :md5, md5 )
-        end
-      end
-    }
-    true
-  end
-
-  def self.fs_check_md5(fix_it = nil)
-    Store::File.where( :file_system => true ).each {|item|
-      md5 = Digest::MD5.hexdigest( item.read_from_fs )
-      if md5 != item.md5
-        puts "DIFF: md5 diff of Store::File.find(#{item.id}) "
-        if fix_it
-          item.update_attribute( :md5, md5 )
-        end
-      end
-    }
-    true
-  end
-
-  def self.move_to_fs
-    Store::File.where( :file_system => false ).each {|item|
-      item.write_to_fs
-      item.update_attribute( :file_system, true )
-      item.update_attribute( :data, nil )
-    }
-  end
-
-  def self.move_to_db
-    Store::File.where( :file_system => true ).each {|item|
-      item.write_to_db
-      item.update_attribute( :file_system, false )
-      item.unlink_location
-    }
-  end
-
-  private
-
-  def check_location
-
-    # write initial to fs if needed
-    if self.file_system && self.data
-      self.write_to_fs
-      self.data = nil
-    end
-  end
-
-  def add_md5
-    if self.data && !self.md5
-      self.md5 = Digest::MD5.hexdigest( self.data )
+  def provider
+    file = Store::File.where( :id => self.store_file_id ).first
+    if !file
+      raise "No such file #{ self.store_file_id }!"
     end
+    file.provider
   end
 end

app/models/store/file.rb

@@ -0,0 +1,82 @@
+# Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/
+
+class Store::File < ApplicationModel
+  include ApplicationLib
+  after_destroy     :destroy_provider
+
+  # add new file
+  def self.add(data)
+    md5 = Digest::MD5.hexdigest( data )
+    file = Store::File.where( :md5 => md5 ).first
+    if file == nil
+
+      # load backend based on config
+      adapter_name = Setting.get('storage_provider') || 'DB'
+      if !adapter_name
+        raise "Missing storage_provider setting option"
+      end
+      adapter = self.load_adapter( "Store::Provider::#{ adapter_name }" )
+      adapter.add( data, md5 )
+      file = Store::File.create(
+        :provider => adapter_name,
+        :md5      => md5,
+      )
+    end
+    file
+  end
+
+  # read content
+  def content
+    puts "get #{self.id} #{self.provider}"
+    adapter = self.class.load_adapter("Store::Provider::#{ self.provider }")
+    adapter.get( self.md5 )
+  end
+
+
+  # check data and md5, in case fix it
+  def self.check_md5(fix_it = nil)
+    success = true
+    Store::File.all.each {|item|
+      content = item.content
+      md5 = Digest::MD5.hexdigest( content )
+      puts "CHECK: Store::File.find(#{item.id}) "
+      if md5 != item.md5
+        success = false
+        puts "DIFF: md5 diff of Store::File.find(#{item.id}) "
+        if fix_it
+          item.update_attribute( :md5, md5 )
+        end
+      end
+    }
+    success
+  end
+
+  # move file from one to other provider
+  def self.move(source, target)
+    adapter_source = load_adapter("Store::Provider::#{ source }")
+    adapter_target = load_adapter("Store::Provider::#{ target }")
+
+    Store::File.all.each {|item|
+      next if item.provider == target
+      content = item.content
+
+      # add to new provider
+      adapter_target.add( content, item.md5 )
+
+      # update meta data
+      item.update_attribute( :provider, target )
+
+      # remove from old provider
+      adapter_source.delete( item.md5 )
+
+      puts "NOTICE: Moved file #{item.md5} from #{source} to #{target}"
+    }
+  end
+
+  private
+
+  def destroy_provider
+    adapter = self.class.load_adapter("Store::Provider::#{ self.provider }")
+    adapter.delete( md5 )
+  end
+end

app/models/store/object.rb

@@ -0,0 +1,5 @@
+# Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/
+
+class Store::Object < ApplicationModel
+  validates :name, :presence => true
+end

app/models/store/provider/db.rb

@@ -0,0 +1,25 @@
+# Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/
+
+class Store::Provider::DB < ApplicationModel
+  self.table_name = 'store_provider_dbs'
+
+  def self.add(data, md5)
+    Store::Provider::DB.create(
+      :data => data,
+      :md5  => md5,
+    )
+    true
+  end
+
+  def self.get(md5)
+    file = Store::Provider::DB.where( :md5 => md5 ).first
+    return if !file
+    file.data
+  end
+
+  def self.delete(md5)
+    Store::Provider::DB.where( :md5 => md5 ).destroy_all
+    true
+  end
+
+end

app/models/store/provider/file.rb

@@ -0,0 +1,84 @@
+# Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/
+
+class Store::Provider::File
+
+  def self.add(data, md5)
+    write_to_fs(data, md5)
+    true
+  end
+
+  def self.get(md5)
+    read_from_fs(md5)
+  end
+
+  def self.delete(md5)
+    unlink_from_fs(md5)
+  end
+
+  private
+
+  # generate file location
+  def self.get_locaton(md5)
+
+    # generate directory
+    base = Rails.root.to_s + '/storage/fs/'
+    parts = md5.scan(/.{1,3}/)
+    path = parts[ 1 .. 7 ].join('/') + '/'
+    file = parts[ 8 .. parts.count ].join('')
+    location = "#{base}/#{path}"
+
+    # create directory if not exists
+    if !File.exist?( location )
+      FileUtils.mkdir_p( location )
+    end
+    location += file
+  end
+
+  # unlink file from fs
+  def self.unlink_from_fs(md5)
+    if File.exist?( get_locaton(md5) )
+      puts "NOTICE: storge remove '#{ get_locaton(md5) }'"
+      File.delete( get_locaton(md5) )
+    end
+  end
+
+  # read file from fs
+  def self.read_from_fs(md5)
+    puts "read from fs #{ get_locaton(md5) }"
+    if !File.exist?( get_locaton(md5) )
+      raise "ERROR: No such file #{ get_locaton(md5) }"
+    end
+    data = File.open( get_locaton(md5), 'rb' )
+    content = data.read
+
+    # check md5
+    local_md5 = Digest::MD5.hexdigest( content )
+    if local_md5 != md5
+      raise "ERROR: Corrupt file in fs #{ get_locaton(md5) }, md5 should be #{md5} but is #{local_md5}"
+    end
+    content
+  end
+
+  # write file to fs
+  def self.write_to_fs(data,md5)
+
+    # install file
+    permission = '600'
+    if !File.exist?( get_locaton(md5) )
+      puts "NOTICE: storge write '#{ get_locaton(md5) }' (#{permission})"
+      file = File.new( get_locaton(md5), 'wb' )
+      file.write( data )
+      file.close
+    end
+    File.chmod( permission.to_i(8), get_locaton(md5) )
+
+    # check md5
+    local_md5 = Digest::MD5.hexdigest( read_from_fs(md5) )
+    if md5 != local_md5
+      raise "ERROR: Corrupt file in fs #{ get_locaton(md5) }, md5 should be #{md5} but is #{local_md5}"
+    end
+
+    true
+  end
+
+end

app/models/ticket/number.rb

@@ -1,6 +1,7 @@
 # Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/
 
-class Ticket::Number < ApplicationLib
+class Ticket::Number
+  include ApplicationLib
 
 =begin
 
@@ -48,7 +49,7 @@ returns
     if !adapter_name
       raise "Missing ticket_number setting option"
     end
-    adapter = self.load_adapter(adapter_name)
+    adapter = load_adapter(adapter_name)
     if !adapter
       raise "Can't load ticket_number adapter '#{adapter_name}'"
     end

db/migrate/20140502000001_update_storage2.rb

@@ -0,0 +1,28 @@
+class UpdateStorage2 < ActiveRecord::Migration
+  def up
+    create_table :store_provider_dbs do |t|
+      t.column :data,     :binary,        :limit => 200.megabytes,  :null => true
+      t.column :md5,      :string,        :limit => 60,             :null => false
+      t.timestamps
+    end
+    add_index :store_provider_dbs, [:md5],   :unique => true
+
+    add_column  :store_files, :provider,    :string,  :limit => 20, :null => true
+    add_index   :store_files, [:provider]
+
+    Store::File.all.each {|file|
+      if file.data
+        file.update_attribute( :provider, 'DB' )
+        Store::Provider::DB.add( file.data, file.md5 )
+      else
+        file.update_attribute( :provider, 'File' )
+        Store::Provider::File.add( file.data, file.md5 )
+      end
+    }
+
+    remove_column :store_files, :data
+  end
+
+  def down
+  end
+end

lib/application_lib.rb

@@ -1,4 +1,9 @@
-class ApplicationLib
+module ApplicationLib
+  def self.included(base)
+    base.extend(ClassMethods)
+  end
+
+  module ClassMethods
 
 =begin
 
@@ -12,13 +17,13 @@ returns
 
 =end
 
-  def self.load_adapter_by_setting(setting)
-    adapter = Setting.get( setting )
-    return if !adapter
+    def load_adapter_by_setting(setting)
+      adapter = Setting.get( setting )
+      return if !adapter
 
-    # load backend
-    self.load_adapter(adapter)
-  end
+      # load backend
+      self.load_adapter(adapter)
+    end
 
 =begin
 
@@ -32,19 +37,26 @@ returns
 
 =end
 
-  def self.load_adapter(adapter)
+    def load_adapter(adapter)
 
-    # load adapter
+      # load adapter
 
-    # will only work on ruby 2.0
-#    Object.const_get(adapter)
+      # will only work on ruby 2.0
+#      Object.const_get(adapter)
 
-    # will work on ruby 1.9 and 2.0
-#    adapter.split('::').inject(Object) do |mod, class_name|
-#      mod.const_get(class_name)
-#    end
+      # will work on ruby 1.9 and 2.0
+#      adapter.split('::').inject(Object) do |mod, class_name|
+#        mod.const_get(class_name)
+#      end
 
-    # will work with active_support
-    adapter.constantize
+      # will work with active_support
+      adapter = adapter.constantize
+
+      if !adapter
+        raise "Can't load adapter '#{adapter_name}'"
+      end
+
+      adapter
+    end
   end
 end

lib/auth.rb

@@ -1,6 +1,7 @@
 # Copyright (C) 2012-2013 Zammad Foundation, http://zammad-foundation.org/
 
-class Auth < ApplicationLib
+class Auth
+  include ApplicationLib
 
 =begin
 

lib/geo_ip.rb

@@ -1,6 +1,7 @@
 # Copyright (C) 2012-2013 Zammad Foundation, http://zammad-foundation.org/
 
-class GeoIp < ApplicationLib
+class GeoIp
+  include ApplicationLib
 
 =begin
 

lib/geo_location.rb

@@ -1,6 +1,7 @@
 # Copyright (C) 2012-2013 Zammad Foundation, http://zammad-foundation.org/
 
-class GeoLocation < ApplicationLib
+class GeoLocation
+  include ApplicationLib
 
 =begin
 

lib/sessions/backend/collections.rb

@@ -27,7 +27,9 @@ module Sessions::Backend::Collections
         end
         push_collection_cache = Sessions::CacheIn.get( cache_key, { :re_expire => true } )
         worker.log 'notice', "---user - fetch push_collection data " + cache_key
-        if !push_collection[key] || !push_collection_cache || push_collection[key] != push_collection_cache || !push_collection[ key ].zip( push_collection_cache ).all? { |x, y| x.attributes == y.attributes }
+#        if !push_collection[key] || !push_collection_cache || push_collection[key] != push_collection_cache || !push_collection[ key ].zip( push_collection_cache ).all? { |x, y| x.attributes == y.attributes }
+        if !push_collection[key] || !push_collection_cache || push_collection[key] != push_collection_cache || !push_collection[ key ].zip( push_collection_cache ).all? { |x, y| return false if !x; return false if !y; x.attributes == y.attributes }
+
           worker.log 'notify', 'fetch push_collection changed - ' + cache_key
           Sessions::CacheIn.set( cache_key, push_collection[key], { :expires_in => 1.minutes } )
         end

lib/sso.rb

@@ -1,6 +1,7 @@
 # Copyright (C) 2012-2013 Zammad Foundation, http://zammad-foundation.org/
 
-class Sso < ApplicationLib
+class Sso
+  include ApplicationLib
 
 =begin
 

test/unit/store_test.rb

@@ -54,9 +54,14 @@ class StoreTest < ActiveSupport::TestCase
       # filename check
       assert_equal( file[:filename], attachments[0].filename )
 
+      # provider check
+      assert_equal( 'DB', attachments[0].provider )
     }
 
-    Store::File.move_to_fs
+    success = Store::File.check_md5
+    assert success, "check_md5 ok"
+
+    Store::File.move( 'DB', 'File' )
 
     files.each { |file|
       md5 = Digest::MD5.hexdigest( file[:data] )
@@ -74,9 +79,15 @@ class StoreTest < ActiveSupport::TestCase
 
       # filename check
       assert_equal( file[:filename], attachments[0].filename )
+
+      # provider check
+      assert_equal( 'File', attachments[0].provider )
     }
 
-    Store::File.move_to_db
+    success = Store::File.check_md5
+    assert success, "check_md5 ok"
+
+    Store::File.move( 'File', 'DB' )
 
     files.each { |file|
       md5 = Digest::MD5.hexdigest( file[:data] )
@@ -95,6 +106,9 @@ class StoreTest < ActiveSupport::TestCase
       # filename check
       assert_equal( file[:filename], attachments[0].filename )
 
+      # provider check
+      assert_equal( 'DB', attachments[0].provider )
+
       # delete attachments
       success = Store.remove(
         :object => 'Test',