diff --git a/app/models/store.rb b/app/models/store.rb index 9d69e21e1..40a7087f4 100644 --- a/app/models/store.rb +++ b/app/models/store.rb @@ -2,7 +2,11 @@ require 'digest/md5' + class Store < ApplicationModel + require 'store/object' + require 'store/file' + store :preferences belongs_to :store_object, :class_name => 'Store::Object' belongs_to :store_file, :class_name => 'Store::File' @@ -35,28 +39,10 @@ returns store_object = Store::Object.create_if_not_exists( :name => data['object'] ) data['store_object_id'] = store_object.id - # check if record already exists - # store = Store.where( :store_object_id => store_object.id, :o_id => data['o_id'], ).first - # if store != nil - # return store - # end + # add to real store + file = Store::File.add( data['data'] ) - # check real store - md5 = Digest::MD5.hexdigest( data['data'] ) data['size'] = data['data'].to_s.bytesize - - # file = Store::Provider::DB.create( data['data'], md5 ) - - file = Store::File.where( :md5 => md5 ).first - - # store attachment - if file == nil - file = Store::File.create( - :data => data['data'], - :md5 => md5, - ) - end - data['store_file_id'] = file.id # not needed attributes @@ -128,7 +114,6 @@ returns # check backend for references files = Store.where( :store_file_id => store.store_file_id ) if files.count == 1 && files.first.id == store.id - # file = Store::Provider::DB.delete( store.store_file_id ) Store::File.find( store.store_file_id ).destroy end @@ -137,165 +122,19 @@ returns return true end - # get attachment def content - # Store::Provider::DB.content( store.store_file_id ) file = Store::File.where( :id => self.store_file_id ).first - return if !file - if file.file_system - return file.read_from_fs + if !file + raise "No such file #{ self.store_file_id }!" end - file.data + file.content end -end -class Store::Object < ApplicationModel - validates :name, :presence => true -end - -class Store::File < ApplicationModel - before_validation :add_md5 - before_create :check_location - after_destroy :unlink_location - - # generate file location - def get_locaton - - # generate directory - base = Rails.root.to_s + '/storage/fs/' - parts = self.md5.scan(/.{1,3}/) - path = parts[ 1 .. 7 ].join('/') + '/' - file = parts[ 8 .. parts.count ].join('') - location = "#{base}/#{path}" - - # create directory if not exists - if !File.exist?( location ) - FileUtils.mkdir_p( location ) - end - location += file - end - - # read file from fs - def unlink_location - if File.exist?( self.get_locaton ) - puts "NOTICE: storge remove '#{self.get_locaton}'" - File.delete( self.get_locaton ) - end - end - - # read file from fs - def read_from_fs - puts "read from fs #{self.get_locaton}" - return if !File.exist?( self.get_locaton ) - data = File.open( self.get_locaton, 'rb' ) - content = data.read - - # check md5 - md5 = Digest::MD5.hexdigest( content ) - if md5 != self.md5 - raise "ERROR: Corrupt file in fs #{self.get_locaton}, md5 should be #{self.md5} but is #{md5}" - end - content - end - - # write file to fs - def write_to_fs - - # install file - permission = '600' - if !File.exist?( self.get_locaton ) - puts "NOTICE: storge write '#{self.get_locaton}' (#{permission})" - file = File.new( self.get_locaton, 'wb' ) - file.write( self.data ) - file.close - end - File.chmod( permission.to_i(8), self.get_locaton ) - - # check md5 - md5 = Digest::MD5.hexdigest( self.read_from_fs ) - if md5 != self.md5 - raise "ERROR: Corrupt file in fs #{self.get_locaton}, md5 should be #{self.md5} but is #{md5}" - end - - true - end - - # write file to db - def write_to_db - - # read and check md5 - content = self.read_from_fs - - # store in database - self.data = content - self.save - - # check md5 against db content - md5 = Digest::MD5.hexdigest( self.data ) - if md5 != self.md5 - raise "ERROR: Corrupt file in db #{self.get_locaton}, md5 should be #{self.md5} but is #{md5}" - end - - true - end - - # check database data and md5, in case fix it - def self.db_check_md5(fix_it = nil) - Store::File.where( :file_system => false ).each {|item| - md5 = Digest::MD5.hexdigest( item.data ) - if md5 != item.md5 - puts "DIFF: md5 diff of Store::File.find(#{item.id}) " - if fix_it - item.update_attribute( :md5, md5 ) - end - end - } - true - end - - def self.fs_check_md5(fix_it = nil) - Store::File.where( :file_system => true ).each {|item| - md5 = Digest::MD5.hexdigest( item.read_from_fs ) - if md5 != item.md5 - puts "DIFF: md5 diff of Store::File.find(#{item.id}) " - if fix_it - item.update_attribute( :md5, md5 ) - end - end - } - true - end - - def self.move_to_fs - Store::File.where( :file_system => false ).each {|item| - item.write_to_fs - item.update_attribute( :file_system, true ) - item.update_attribute( :data, nil ) - } - end - - def self.move_to_db - Store::File.where( :file_system => true ).each {|item| - item.write_to_db - item.update_attribute( :file_system, false ) - item.unlink_location - } - end - - private - - def check_location - - # write initial to fs if needed - if self.file_system && self.data - self.write_to_fs - self.data = nil - end - end - - def add_md5 - if self.data && !self.md5 - self.md5 = Digest::MD5.hexdigest( self.data ) + def provider + file = Store::File.where( :id => self.store_file_id ).first + if !file + raise "No such file #{ self.store_file_id }!" end + file.provider end end \ No newline at end of file diff --git a/app/models/store/file.rb b/app/models/store/file.rb new file mode 100644 index 000000000..ee7257eb8 --- /dev/null +++ b/app/models/store/file.rb @@ -0,0 +1,82 @@ +# Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/ + +class Store::File < ApplicationModel + include ApplicationLib + after_destroy :destroy_provider + + # add new file + def self.add(data) + md5 = Digest::MD5.hexdigest( data ) + file = Store::File.where( :md5 => md5 ).first + if file == nil + + # load backend based on config + adapter_name = Setting.get('storage_provider') || 'DB' + if !adapter_name + raise "Missing storage_provider setting option" + end + adapter = self.load_adapter( "Store::Provider::#{ adapter_name }" ) + adapter.add( data, md5 ) + file = Store::File.create( + :provider => adapter_name, + :md5 => md5, + ) + end + file + end + + # read content + def content + puts "get #{self.id} #{self.provider}" + adapter = self.class.load_adapter("Store::Provider::#{ self.provider }") + adapter.get( self.md5 ) + end + + + # check data and md5, in case fix it + def self.check_md5(fix_it = nil) + success = true + Store::File.all.each {|item| + content = item.content + md5 = Digest::MD5.hexdigest( content ) + puts "CHECK: Store::File.find(#{item.id}) " + if md5 != item.md5 + success = false + puts "DIFF: md5 diff of Store::File.find(#{item.id}) " + if fix_it + item.update_attribute( :md5, md5 ) + end + end + } + success + end + + # move file from one to other provider + def self.move(source, target) + adapter_source = load_adapter("Store::Provider::#{ source }") + adapter_target = load_adapter("Store::Provider::#{ target }") + + Store::File.all.each {|item| + next if item.provider == target + content = item.content + + # add to new provider + adapter_target.add( content, item.md5 ) + + # update meta data + item.update_attribute( :provider, target ) + + # remove from old provider + adapter_source.delete( item.md5 ) + + puts "NOTICE: Moved file #{item.md5} from #{source} to #{target}" + } + end + + private + + def destroy_provider + adapter = self.class.load_adapter("Store::Provider::#{ self.provider }") + adapter.delete( md5 ) + end +end \ No newline at end of file diff --git a/app/models/store/object.rb b/app/models/store/object.rb new file mode 100644 index 000000000..8d2f11c67 --- /dev/null +++ b/app/models/store/object.rb @@ -0,0 +1,5 @@ +# Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/ + +class Store::Object < ApplicationModel + validates :name, :presence => true +end \ No newline at end of file diff --git a/app/models/store/provider/db.rb b/app/models/store/provider/db.rb new file mode 100644 index 000000000..bc31a8e84 --- /dev/null +++ b/app/models/store/provider/db.rb @@ -0,0 +1,25 @@ +# Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/ + +class Store::Provider::DB < ApplicationModel + self.table_name = 'store_provider_dbs' + + def self.add(data, md5) + Store::Provider::DB.create( + :data => data, + :md5 => md5, + ) + true + end + + def self.get(md5) + file = Store::Provider::DB.where( :md5 => md5 ).first + return if !file + file.data + end + + def self.delete(md5) + Store::Provider::DB.where( :md5 => md5 ).destroy_all + true + end + +end \ No newline at end of file diff --git a/app/models/store/provider/file.rb b/app/models/store/provider/file.rb new file mode 100644 index 000000000..71b92c213 --- /dev/null +++ b/app/models/store/provider/file.rb @@ -0,0 +1,84 @@ +# Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/ + +class Store::Provider::File + + def self.add(data, md5) + write_to_fs(data, md5) + true + end + + def self.get(md5) + read_from_fs(md5) + end + + def self.delete(md5) + unlink_from_fs(md5) + end + + private + + # generate file location + def self.get_locaton(md5) + + # generate directory + base = Rails.root.to_s + '/storage/fs/' + parts = md5.scan(/.{1,3}/) + path = parts[ 1 .. 7 ].join('/') + '/' + file = parts[ 8 .. parts.count ].join('') + location = "#{base}/#{path}" + + # create directory if not exists + if !File.exist?( location ) + FileUtils.mkdir_p( location ) + end + location += file + end + + # unlink file from fs + def self.unlink_from_fs(md5) + if File.exist?( get_locaton(md5) ) + puts "NOTICE: storge remove '#{ get_locaton(md5) }'" + File.delete( get_locaton(md5) ) + end + end + + # read file from fs + def self.read_from_fs(md5) + puts "read from fs #{ get_locaton(md5) }" + if !File.exist?( get_locaton(md5) ) + raise "ERROR: No such file #{ get_locaton(md5) }" + end + data = File.open( get_locaton(md5), 'rb' ) + content = data.read + + # check md5 + local_md5 = Digest::MD5.hexdigest( content ) + if local_md5 != md5 + raise "ERROR: Corrupt file in fs #{ get_locaton(md5) }, md5 should be #{md5} but is #{local_md5}" + end + content + end + + # write file to fs + def self.write_to_fs(data,md5) + + # install file + permission = '600' + if !File.exist?( get_locaton(md5) ) + puts "NOTICE: storge write '#{ get_locaton(md5) }' (#{permission})" + file = File.new( get_locaton(md5), 'wb' ) + file.write( data ) + file.close + end + File.chmod( permission.to_i(8), get_locaton(md5) ) + + # check md5 + local_md5 = Digest::MD5.hexdigest( read_from_fs(md5) ) + if md5 != local_md5 + raise "ERROR: Corrupt file in fs #{ get_locaton(md5) }, md5 should be #{md5} but is #{local_md5}" + end + + true + end + +end \ No newline at end of file diff --git a/app/models/ticket/number.rb b/app/models/ticket/number.rb index 05c50c1f6..864a4c67a 100644 --- a/app/models/ticket/number.rb +++ b/app/models/ticket/number.rb @@ -1,6 +1,7 @@ # Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/ -class Ticket::Number < ApplicationLib +class Ticket::Number + include ApplicationLib =begin @@ -48,7 +49,7 @@ returns if !adapter_name raise "Missing ticket_number setting option" end - adapter = self.load_adapter(adapter_name) + adapter = load_adapter(adapter_name) if !adapter raise "Can't load ticket_number adapter '#{adapter_name}'" end diff --git a/db/migrate/20140502000001_update_storage2.rb b/db/migrate/20140502000001_update_storage2.rb new file mode 100644 index 000000000..0f349d875 --- /dev/null +++ b/db/migrate/20140502000001_update_storage2.rb @@ -0,0 +1,28 @@ +class UpdateStorage2 < ActiveRecord::Migration + def up + create_table :store_provider_dbs do |t| + t.column :data, :binary, :limit => 200.megabytes, :null => true + t.column :md5, :string, :limit => 60, :null => false + t.timestamps + end + add_index :store_provider_dbs, [:md5], :unique => true + + add_column :store_files, :provider, :string, :limit => 20, :null => true + add_index :store_files, [:provider] + + Store::File.all.each {|file| + if file.data + file.update_attribute( :provider, 'DB' ) + Store::Provider::DB.add( file.data, file.md5 ) + else + file.update_attribute( :provider, 'File' ) + Store::Provider::File.add( file.data, file.md5 ) + end + } + + remove_column :store_files, :data + end + + def down + end +end diff --git a/lib/application_lib.rb b/lib/application_lib.rb index 142743394..4ea188f2c 100644 --- a/lib/application_lib.rb +++ b/lib/application_lib.rb @@ -1,4 +1,9 @@ -class ApplicationLib +module ApplicationLib + def self.included(base) + base.extend(ClassMethods) + end + + module ClassMethods =begin @@ -12,13 +17,13 @@ returns =end - def self.load_adapter_by_setting(setting) - adapter = Setting.get( setting ) - return if !adapter + def load_adapter_by_setting(setting) + adapter = Setting.get( setting ) + return if !adapter - # load backend - self.load_adapter(adapter) - end + # load backend + self.load_adapter(adapter) + end =begin @@ -32,19 +37,26 @@ returns =end - def self.load_adapter(adapter) + def load_adapter(adapter) - # load adapter + # load adapter - # will only work on ruby 2.0 -# Object.const_get(adapter) + # will only work on ruby 2.0 +# Object.const_get(adapter) - # will work on ruby 1.9 and 2.0 -# adapter.split('::').inject(Object) do |mod, class_name| -# mod.const_get(class_name) -# end + # will work on ruby 1.9 and 2.0 +# adapter.split('::').inject(Object) do |mod, class_name| +# mod.const_get(class_name) +# end - # will work with active_support - adapter.constantize + # will work with active_support + adapter = adapter.constantize + + if !adapter + raise "Can't load adapter '#{adapter_name}'" + end + + adapter + end end end diff --git a/lib/auth.rb b/lib/auth.rb index 046dd3599..1a8918d50 100644 --- a/lib/auth.rb +++ b/lib/auth.rb @@ -1,6 +1,7 @@ # Copyright (C) 2012-2013 Zammad Foundation, http://zammad-foundation.org/ -class Auth < ApplicationLib +class Auth + include ApplicationLib =begin diff --git a/lib/geo_ip.rb b/lib/geo_ip.rb index 098b00e03..2c3885069 100644 --- a/lib/geo_ip.rb +++ b/lib/geo_ip.rb @@ -1,6 +1,7 @@ # Copyright (C) 2012-2013 Zammad Foundation, http://zammad-foundation.org/ -class GeoIp < ApplicationLib +class GeoIp + include ApplicationLib =begin diff --git a/lib/geo_location.rb b/lib/geo_location.rb index c79608f47..b18df5560 100644 --- a/lib/geo_location.rb +++ b/lib/geo_location.rb @@ -1,6 +1,7 @@ # Copyright (C) 2012-2013 Zammad Foundation, http://zammad-foundation.org/ -class GeoLocation < ApplicationLib +class GeoLocation + include ApplicationLib =begin diff --git a/lib/sessions/backend/collections.rb b/lib/sessions/backend/collections.rb index 2eaf6ab11..672afb773 100644 --- a/lib/sessions/backend/collections.rb +++ b/lib/sessions/backend/collections.rb @@ -27,7 +27,9 @@ module Sessions::Backend::Collections end push_collection_cache = Sessions::CacheIn.get( cache_key, { :re_expire => true } ) worker.log 'notice', "---user - fetch push_collection data " + cache_key - if !push_collection[key] || !push_collection_cache || push_collection[key] != push_collection_cache || !push_collection[ key ].zip( push_collection_cache ).all? { |x, y| x.attributes == y.attributes } +# if !push_collection[key] || !push_collection_cache || push_collection[key] != push_collection_cache || !push_collection[ key ].zip( push_collection_cache ).all? { |x, y| x.attributes == y.attributes } + if !push_collection[key] || !push_collection_cache || push_collection[key] != push_collection_cache || !push_collection[ key ].zip( push_collection_cache ).all? { |x, y| return false if !x; return false if !y; x.attributes == y.attributes } + worker.log 'notify', 'fetch push_collection changed - ' + cache_key Sessions::CacheIn.set( cache_key, push_collection[key], { :expires_in => 1.minutes } ) end diff --git a/lib/sso.rb b/lib/sso.rb index 858afc28f..67ef0a9f5 100644 --- a/lib/sso.rb +++ b/lib/sso.rb @@ -1,6 +1,7 @@ # Copyright (C) 2012-2013 Zammad Foundation, http://zammad-foundation.org/ -class Sso < ApplicationLib +class Sso + include ApplicationLib =begin diff --git a/test/unit/store_test.rb b/test/unit/store_test.rb index 92f62a2b0..f380f7bcc 100644 --- a/test/unit/store_test.rb +++ b/test/unit/store_test.rb @@ -54,9 +54,14 @@ class StoreTest < ActiveSupport::TestCase # filename check assert_equal( file[:filename], attachments[0].filename ) + # provider check + assert_equal( 'DB', attachments[0].provider ) } - Store::File.move_to_fs + success = Store::File.check_md5 + assert success, "check_md5 ok" + + Store::File.move( 'DB', 'File' ) files.each { |file| md5 = Digest::MD5.hexdigest( file[:data] ) @@ -74,9 +79,15 @@ class StoreTest < ActiveSupport::TestCase # filename check assert_equal( file[:filename], attachments[0].filename ) + + # provider check + assert_equal( 'File', attachments[0].provider ) } - Store::File.move_to_db + success = Store::File.check_md5 + assert success, "check_md5 ok" + + Store::File.move( 'File', 'DB' ) files.each { |file| md5 = Digest::MD5.hexdigest( file[:data] ) @@ -95,6 +106,9 @@ class StoreTest < ActiveSupport::TestCase # filename check assert_equal( file[:filename], attachments[0].filename ) + # provider check + assert_equal( 'DB', attachments[0].provider ) + # delete attachments success = Store.remove( :object => 'Test',