From 84c23e2311494daac3dbbc188400ce91706174a8 Mon Sep 17 00:00:00 2001 From: Martin Edenhofer Date: Sat, 3 May 2014 16:52:37 +0200 Subject: [PATCH] Moved from md5 to sha256 for file identification. --- app/models/store.rb | 3 - app/models/store/file.rb | 34 +++++------ app/models/store/provider/db.rb | 12 ++-- app/models/store/provider/file.rb | 64 ++++++++++---------- db/migrate/20140503000001_update_storage3.rb | 27 +++++++++ test/unit/store_test.rb | 32 +++++----- 6 files changed, 98 insertions(+), 74 deletions(-) create mode 100644 db/migrate/20140503000001_update_storage3.rb diff --git a/app/models/store.rb b/app/models/store.rb index 40a7087f4..e416a9358 100644 --- a/app/models/store.rb +++ b/app/models/store.rb @@ -1,8 +1,5 @@ # Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/ -require 'digest/md5' - - class Store < ApplicationModel require 'store/object' require 'store/file' diff --git a/app/models/store/file.rb b/app/models/store/file.rb index ee7257eb8..bc9b6c6a2 100644 --- a/app/models/store/file.rb +++ b/app/models/store/file.rb @@ -6,8 +6,9 @@ class Store::File < ApplicationModel # add new file def self.add(data) - md5 = Digest::MD5.hexdigest( data ) - file = Store::File.where( :md5 => md5 ).first + sha = Digest::SHA256.hexdigest( data ) + + file = Store::File.where( :sha => sha ).first if file == nil # load backend based on config @@ -16,10 +17,10 @@ class Store::File < ApplicationModel raise "Missing storage_provider setting option" end adapter = self.load_adapter( "Store::Provider::#{ adapter_name }" ) - adapter.add( data, md5 ) + adapter.add( data, sha ) file = Store::File.create( :provider => adapter_name, - :md5 => md5, + :sha => sha, ) end file @@ -27,24 +28,22 @@ class Store::File < ApplicationModel # read content def content - puts "get #{self.id} #{self.provider}" adapter = self.class.load_adapter("Store::Provider::#{ self.provider }") - adapter.get( self.md5 ) + adapter.get( self.sha ) end - - # check data and md5, in case fix it - def self.check_md5(fix_it = nil) + # check data and sha, in case fix it + def self.verify(fix_it = nil) success = true Store::File.all.each {|item| content = item.content - md5 = Digest::MD5.hexdigest( content ) + sha = Digest::SHA256.hexdigest( content ) puts "CHECK: Store::File.find(#{item.id}) " - if md5 != item.md5 + if sha != item.sha success = false - puts "DIFF: md5 diff of Store::File.find(#{item.id}) " + puts "DIFF: sha diff of Store::File.find(#{item.id}) " if fix_it - item.update_attribute( :md5, md5 ) + item.update_attribute( :sha, sha ) end end } @@ -61,22 +60,23 @@ class Store::File < ApplicationModel content = item.content # add to new provider - adapter_target.add( content, item.md5 ) + adapter_target.add( content, item.sha ) # update meta data item.update_attribute( :provider, target ) # remove from old provider - adapter_source.delete( item.md5 ) + adapter_source.delete( item.sha ) - puts "NOTICE: Moved file #{item.md5} from #{source} to #{target}" + puts "NOTICE: Moved file #{item.sha} from #{source} to #{target}" } + true end private def destroy_provider adapter = self.class.load_adapter("Store::Provider::#{ self.provider }") - adapter.delete( md5 ) + adapter.delete( self.sha ) end end \ No newline at end of file diff --git a/app/models/store/provider/db.rb b/app/models/store/provider/db.rb index bc31a8e84..fc8cd462d 100644 --- a/app/models/store/provider/db.rb +++ b/app/models/store/provider/db.rb @@ -3,22 +3,22 @@ class Store::Provider::DB < ApplicationModel self.table_name = 'store_provider_dbs' - def self.add(data, md5) + def self.add(data, sha) Store::Provider::DB.create( :data => data, - :md5 => md5, + :sha => sha, ) true end - def self.get(md5) - file = Store::Provider::DB.where( :md5 => md5 ).first + def self.get(sha) + file = Store::Provider::DB.where( :sha => sha ).first return if !file file.data end - def self.delete(md5) - Store::Provider::DB.where( :md5 => md5 ).destroy_all + def self.delete(sha) + Store::Provider::DB.where( :sha => sha ).destroy_all true end diff --git a/app/models/store/provider/file.rb b/app/models/store/provider/file.rb index 71b92c213..35c2c98c7 100644 --- a/app/models/store/provider/file.rb +++ b/app/models/store/provider/file.rb @@ -2,29 +2,29 @@ class Store::Provider::File - def self.add(data, md5) - write_to_fs(data, md5) + def self.add(data, sha) + write_to_fs(data, sha) true end - def self.get(md5) - read_from_fs(md5) + def self.get(sha) + read_from_fs(sha) end - def self.delete(md5) - unlink_from_fs(md5) + def self.delete(sha) + unlink_from_fs(sha) end private # generate file location - def self.get_locaton(md5) + def self.get_locaton(sha) # generate directory base = Rails.root.to_s + '/storage/fs/' - parts = md5.scan(/.{1,3}/) - path = parts[ 1 .. 7 ].join('/') + '/' - file = parts[ 8 .. parts.count ].join('') + parts = sha.scan(/.{1,3}/) + path = parts[ 1 .. 10 ].join('/') + '/' + file = parts[ 11 .. parts.count ].join('') location = "#{base}/#{path}" # create directory if not exists @@ -35,47 +35,47 @@ class Store::Provider::File end # unlink file from fs - def self.unlink_from_fs(md5) - if File.exist?( get_locaton(md5) ) - puts "NOTICE: storge remove '#{ get_locaton(md5) }'" - File.delete( get_locaton(md5) ) + def self.unlink_from_fs(sha) + if File.exist?( get_locaton(sha) ) + puts "NOTICE: storge remove '#{ get_locaton(sha) }'" + File.delete( get_locaton(sha) ) end end # read file from fs - def self.read_from_fs(md5) - puts "read from fs #{ get_locaton(md5) }" - if !File.exist?( get_locaton(md5) ) - raise "ERROR: No such file #{ get_locaton(md5) }" + def self.read_from_fs(sha) + puts "read from fs #{ get_locaton(sha) }" + if !File.exist?( get_locaton(sha) ) + raise "ERROR: No such file #{ get_locaton(sha) }" end - data = File.open( get_locaton(md5), 'rb' ) + data = File.open( get_locaton(sha), 'rb' ) content = data.read - # check md5 - local_md5 = Digest::MD5.hexdigest( content ) - if local_md5 != md5 - raise "ERROR: Corrupt file in fs #{ get_locaton(md5) }, md5 should be #{md5} but is #{local_md5}" + # check sha + local_sha = Digest::SHA256.hexdigest( content ) + if local_sha != sha + raise "ERROR: Corrupt file in fs #{ get_locaton(sha) }, sha should be #{sha} but is #{local_sha}" end content end # write file to fs - def self.write_to_fs(data,md5) + def self.write_to_fs(data,sha) # install file permission = '600' - if !File.exist?( get_locaton(md5) ) - puts "NOTICE: storge write '#{ get_locaton(md5) }' (#{permission})" - file = File.new( get_locaton(md5), 'wb' ) + if !File.exist?( get_locaton(sha) ) + puts "NOTICE: storge write '#{ get_locaton(sha) }' (#{permission})" + file = File.new( get_locaton(sha), 'wb' ) file.write( data ) file.close end - File.chmod( permission.to_i(8), get_locaton(md5) ) + File.chmod( permission.to_i(8), get_locaton(sha) ) - # check md5 - local_md5 = Digest::MD5.hexdigest( read_from_fs(md5) ) - if md5 != local_md5 - raise "ERROR: Corrupt file in fs #{ get_locaton(md5) }, md5 should be #{md5} but is #{local_md5}" + # check sha + local_sha = Digest::SHA256.hexdigest( read_from_fs(sha) ) + if sha != local_sha + raise "ERROR: Corrupt file in fs #{ get_locaton(sha) }, sha should be #{sha} but is #{local_sha}" end true diff --git a/db/migrate/20140503000001_update_storage3.rb b/db/migrate/20140503000001_update_storage3.rb new file mode 100644 index 000000000..9f5e5263e --- /dev/null +++ b/db/migrate/20140503000001_update_storage3.rb @@ -0,0 +1,27 @@ +class UpdateStorage3 < ActiveRecord::Migration + def up + + add_column :store_files, :sha, :string, :limit => 128, :null => true + add_index :store_files, [:sha], :unique => true + + add_column :store_provider_dbs, :sha, :string, :limit => 128, :null => true + add_index :store_provider_dbs, [:sha], :unique => true + + Store::File.all.each {|file| + next if file.sha + sha = Digest::SHA256.hexdigest( file.content ) + file.update_attribute( :sha, sha ) + } + Store::Provider::DB.all.each {|file| + next if file.sha + sha = Digest::SHA256.hexdigest( file.content ) + file.update_attribute( :sha, sha ) + } + + remove_column :store_files, :md5 + remove_column :store_provider_dbs, :md5 + end + + def down + end +end diff --git a/test/unit/store_test.rb b/test/unit/store_test.rb index f380f7bcc..1b505aaea 100644 --- a/test/unit/store_test.rb +++ b/test/unit/store_test.rb @@ -27,7 +27,7 @@ class StoreTest < ActiveSupport::TestCase ] files.each { |file| - md5 = Digest::MD5.hexdigest( file[:data] ) + sha = Digest::SHA256.hexdigest( file[:data] ) # add attachments store = Store.add( @@ -47,9 +47,9 @@ class StoreTest < ActiveSupport::TestCase ) assert attachments - # md5 check - md5_new = Digest::MD5.hexdigest( attachments[0].content ) - assert_equal( md5, md5_new, "check file #{ file[:filename] }") + # sha check + sha_new = Digest::SHA256.hexdigest( attachments[0].content ) + assert_equal( sha, sha_new, "check file #{ file[:filename] }") # filename check assert_equal( file[:filename], attachments[0].filename ) @@ -58,13 +58,13 @@ class StoreTest < ActiveSupport::TestCase assert_equal( 'DB', attachments[0].provider ) } - success = Store::File.check_md5 - assert success, "check_md5 ok" + success = Store::File.verify + assert success, "verify ok" Store::File.move( 'DB', 'File' ) files.each { |file| - md5 = Digest::MD5.hexdigest( file[:data] ) + sha = Digest::SHA256.hexdigest( file[:data] ) # get list of attachments attachments = Store.list( @@ -73,9 +73,9 @@ class StoreTest < ActiveSupport::TestCase ) assert attachments - # md5 check - md5_new = Digest::MD5.hexdigest( attachments[0].content ) - assert_equal( md5, md5_new, "check file #{ file[:filename] }") + # sha check + sha_new = Digest::SHA256.hexdigest( attachments[0].content ) + assert_equal( sha, sha_new, "check file #{ file[:filename] }") # filename check assert_equal( file[:filename], attachments[0].filename ) @@ -84,13 +84,13 @@ class StoreTest < ActiveSupport::TestCase assert_equal( 'File', attachments[0].provider ) } - success = Store::File.check_md5 - assert success, "check_md5 ok" + success = Store::File.verify + assert success, "verify ok" Store::File.move( 'File', 'DB' ) files.each { |file| - md5 = Digest::MD5.hexdigest( file[:data] ) + sha = Digest::SHA256.hexdigest( file[:data] ) # get list of attachments attachments = Store.list( @@ -99,9 +99,9 @@ class StoreTest < ActiveSupport::TestCase ) assert attachments - # md5 check - md5_new = Digest::MD5.hexdigest( attachments[0].content ) - assert_equal( md5, md5_new, "check file #{ file[:filename] }") + # sha check + sha_new = Digest::SHA256.hexdigest( attachments[0].content ) + assert_equal( sha, sha_new, "check file #{ file[:filename] }") # filename check assert_equal( file[:filename], attachments[0].filename )