Moved from md5 to sha256 for file identification.

app/models/store.rb

@@ -1,8 +1,5 @@
 # Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/
 
-require 'digest/md5'
-
-
 class Store < ApplicationModel
   require 'store/object'
   require 'store/file'

app/models/store/file.rb

@@ -6,8 +6,9 @@ class Store::File < ApplicationModel
 
   # add new file
   def self.add(data)
-    md5 = Digest::MD5.hexdigest( data )
+    sha = Digest::SHA256.hexdigest( data )
-    file = Store::File.where( :md5 => md5 ).first
+
+    file = Store::File.where( :sha => sha ).first
     if file == nil
 
       # load backend based on config
@@ -16,10 +17,10 @@ class Store::File < ApplicationModel
         raise "Missing storage_provider setting option"
       end
       adapter = self.load_adapter( "Store::Provider::#{ adapter_name }" )
-      adapter.add( data, md5 )
+      adapter.add( data, sha )
       file = Store::File.create(
         :provider => adapter_name,
-        :md5      => md5,
+        :sha      => sha,
       )
     end
     file
@@ -27,24 +28,22 @@ class Store::File < ApplicationModel
 
   # read content
   def content
-    puts "get #{self.id} #{self.provider}"
     adapter = self.class.load_adapter("Store::Provider::#{ self.provider }")
-    adapter.get( self.md5 )
+    adapter.get( self.sha )
   end
 
-
+  # check data and sha, in case fix it
-  # check data and md5, in case fix it
+  def self.verify(fix_it = nil)
-  def self.check_md5(fix_it = nil)
     success = true
     Store::File.all.each {|item|
       content = item.content
-      md5 = Digest::MD5.hexdigest( content )
+      sha = Digest::SHA256.hexdigest( content )
       puts "CHECK: Store::File.find(#{item.id}) "
-      if md5 != item.md5
+      if sha != item.sha
         success = false
-        puts "DIFF: md5 diff of Store::File.find(#{item.id}) "
+        puts "DIFF: sha diff of Store::File.find(#{item.id}) "
         if fix_it
-          item.update_attribute( :md5, md5 )
+          item.update_attribute( :sha, sha )
         end
       end
     }
@@ -61,22 +60,23 @@ class Store::File < ApplicationModel
       content = item.content
 
       # add to new provider
-      adapter_target.add( content, item.md5 )
+      adapter_target.add( content, item.sha )
 
       # update meta data
       item.update_attribute( :provider, target )
 
       # remove from old provider
-      adapter_source.delete( item.md5 )
+      adapter_source.delete( item.sha )
 
-      puts "NOTICE: Moved file #{item.md5} from #{source} to #{target}"
+      puts "NOTICE: Moved file #{item.sha} from #{source} to #{target}"
     }
+    true
   end
 
   private
 
   def destroy_provider
     adapter = self.class.load_adapter("Store::Provider::#{ self.provider }")
-    adapter.delete( md5 )
+    adapter.delete( self.sha )
   end
 end

app/models/store/provider/db.rb

@@ -3,22 +3,22 @@
 class Store::Provider::DB < ApplicationModel
   self.table_name = 'store_provider_dbs'
 
-  def self.add(data, md5)
+  def self.add(data, sha)
     Store::Provider::DB.create(
       :data => data,
-      :md5  => md5,
+      :sha  => sha,
     )
     true
   end
 
-  def self.get(md5)
+  def self.get(sha)
-    file = Store::Provider::DB.where( :md5 => md5 ).first
+    file = Store::Provider::DB.where( :sha => sha ).first
     return if !file
     file.data
   end
 
-  def self.delete(md5)
+  def self.delete(sha)
-    Store::Provider::DB.where( :md5 => md5 ).destroy_all
+    Store::Provider::DB.where( :sha => sha ).destroy_all
     true
   end
 

app/models/store/provider/file.rb

@@ -2,29 +2,29 @@
 
 class Store::Provider::File
 
-  def self.add(data, md5)
+  def self.add(data, sha)
-    write_to_fs(data, md5)
+    write_to_fs(data, sha)
     true
   end
 
-  def self.get(md5)
+  def self.get(sha)
-    read_from_fs(md5)
+    read_from_fs(sha)
   end
 
-  def self.delete(md5)
+  def self.delete(sha)
-    unlink_from_fs(md5)
+    unlink_from_fs(sha)
   end
 
   private
 
   # generate file location
-  def self.get_locaton(md5)
+  def self.get_locaton(sha)
 
     # generate directory
     base = Rails.root.to_s + '/storage/fs/'
-    parts = md5.scan(/.{1,3}/)
+    parts = sha.scan(/.{1,3}/)
-    path = parts[ 1 .. 7 ].join('/') + '/'
+    path = parts[ 1 .. 10 ].join('/') + '/'
-    file = parts[ 8 .. parts.count ].join('')
+    file = parts[ 11 .. parts.count ].join('')
     location = "#{base}/#{path}"
 
     # create directory if not exists
@@ -35,47 +35,47 @@ class Store::Provider::File
   end
 
   # unlink file from fs
-  def self.unlink_from_fs(md5)
+  def self.unlink_from_fs(sha)
-    if File.exist?( get_locaton(md5) )
+    if File.exist?( get_locaton(sha) )
-      puts "NOTICE: storge remove '#{ get_locaton(md5) }'"
+      puts "NOTICE: storge remove '#{ get_locaton(sha) }'"
-      File.delete( get_locaton(md5) )
+      File.delete( get_locaton(sha) )
     end
   end
 
   # read file from fs
-  def self.read_from_fs(md5)
+  def self.read_from_fs(sha)
-    puts "read from fs #{ get_locaton(md5) }"
+    puts "read from fs #{ get_locaton(sha) }"
-    if !File.exist?( get_locaton(md5) )
+    if !File.exist?( get_locaton(sha) )
-      raise "ERROR: No such file #{ get_locaton(md5) }"
+      raise "ERROR: No such file #{ get_locaton(sha) }"
     end
-    data = File.open( get_locaton(md5), 'rb' )
+    data = File.open( get_locaton(sha), 'rb' )
     content = data.read
 
-    # check md5
+    # check sha
-    local_md5 = Digest::MD5.hexdigest( content )
+    local_sha = Digest::SHA256.hexdigest( content )
-    if local_md5 != md5
+    if local_sha != sha
-      raise "ERROR: Corrupt file in fs #{ get_locaton(md5) }, md5 should be #{md5} but is #{local_md5}"
+      raise "ERROR: Corrupt file in fs #{ get_locaton(sha) }, sha should be #{sha} but is #{local_sha}"
     end
     content
   end
 
   # write file to fs
-  def self.write_to_fs(data,md5)
+  def self.write_to_fs(data,sha)
 
     # install file
     permission = '600'
-    if !File.exist?( get_locaton(md5) )
+    if !File.exist?( get_locaton(sha) )
-      puts "NOTICE: storge write '#{ get_locaton(md5) }' (#{permission})"
+      puts "NOTICE: storge write '#{ get_locaton(sha) }' (#{permission})"
-      file = File.new( get_locaton(md5), 'wb' )
+      file = File.new( get_locaton(sha), 'wb' )
       file.write( data )
       file.close
     end
-    File.chmod( permission.to_i(8), get_locaton(md5) )
+    File.chmod( permission.to_i(8), get_locaton(sha) )
 
-    # check md5
+    # check sha
-    local_md5 = Digest::MD5.hexdigest( read_from_fs(md5) )
+    local_sha = Digest::SHA256.hexdigest( read_from_fs(sha) )
-    if md5 != local_md5
+    if sha != local_sha
-      raise "ERROR: Corrupt file in fs #{ get_locaton(md5) }, md5 should be #{md5} but is #{local_md5}"
+      raise "ERROR: Corrupt file in fs #{ get_locaton(sha) }, sha should be #{sha} but is #{local_sha}"
     end
 
     true

db/migrate/20140503000001_update_storage3.rb

@@ -0,0 +1,27 @@
+class UpdateStorage3 < ActiveRecord::Migration
+  def up
+
+    add_column  :store_files, :sha,    :string,  :limit => 128, :null => true
+    add_index   :store_files, [:sha],  :unique => true
+
+    add_column  :store_provider_dbs, :sha,    :string,  :limit => 128, :null => true
+    add_index   :store_provider_dbs, [:sha],  :unique => true
+
+    Store::File.all.each {|file|
+      next if file.sha
+      sha = Digest::SHA256.hexdigest( file.content )
+      file.update_attribute( :sha, sha )
+    }
+    Store::Provider::DB.all.each {|file|
+      next if file.sha
+      sha = Digest::SHA256.hexdigest( file.content )
+      file.update_attribute( :sha, sha )
+    }
+
+    remove_column :store_files, :md5
+    remove_column :store_provider_dbs, :md5
+  end
+
+  def down
+  end
+end

test/unit/store_test.rb

@@ -27,7 +27,7 @@ class StoreTest < ActiveSupport::TestCase
     ]
 
     files.each { |file|
-      md5 = Digest::MD5.hexdigest( file[:data] )
+      sha = Digest::SHA256.hexdigest( file[:data] )
 
       # add attachments
       store = Store.add(
@@ -47,9 +47,9 @@ class StoreTest < ActiveSupport::TestCase
       )
       assert attachments
 
-      # md5 check
+      # sha check
-      md5_new = Digest::MD5.hexdigest( attachments[0].content )
+      sha_new = Digest::SHA256.hexdigest( attachments[0].content )
-      assert_equal( md5, md5_new,  "check file #{ file[:filename] }")
+      assert_equal( sha, sha_new,  "check file #{ file[:filename] }")
 
       # filename check
       assert_equal( file[:filename], attachments[0].filename )
@@ -58,13 +58,13 @@ class StoreTest < ActiveSupport::TestCase
       assert_equal( 'DB', attachments[0].provider )
     }
 
-    success = Store::File.check_md5
+    success = Store::File.verify
-    assert success, "check_md5 ok"
+    assert success, "verify ok"
 
     Store::File.move( 'DB', 'File' )
 
     files.each { |file|
-      md5 = Digest::MD5.hexdigest( file[:data] )
+      sha = Digest::SHA256.hexdigest( file[:data] )
 
       # get list of attachments
       attachments = Store.list(
@@ -73,9 +73,9 @@ class StoreTest < ActiveSupport::TestCase
       )
       assert attachments
 
-      # md5 check
+      # sha check
-      md5_new = Digest::MD5.hexdigest( attachments[0].content )
+      sha_new = Digest::SHA256.hexdigest( attachments[0].content )
-      assert_equal( md5, md5_new,  "check file #{ file[:filename] }")
+      assert_equal( sha, sha_new,  "check file #{ file[:filename] }")
 
       # filename check
       assert_equal( file[:filename], attachments[0].filename )
@@ -84,13 +84,13 @@ class StoreTest < ActiveSupport::TestCase
       assert_equal( 'File', attachments[0].provider )
     }
 
-    success = Store::File.check_md5
+    success = Store::File.verify
-    assert success, "check_md5 ok"
+    assert success, "verify ok"
 
     Store::File.move( 'File', 'DB' )
 
     files.each { |file|
-      md5 = Digest::MD5.hexdigest( file[:data] )
+      sha = Digest::SHA256.hexdigest( file[:data] )
 
       # get list of attachments
       attachments = Store.list(
@@ -99,9 +99,9 @@ class StoreTest < ActiveSupport::TestCase
       )
       assert attachments
 
-      # md5 check
+      # sha check
-      md5_new = Digest::MD5.hexdigest( attachments[0].content )
+      sha_new = Digest::SHA256.hexdigest( attachments[0].content )
-      assert_equal( md5, md5_new,  "check file #{ file[:filename] }")
+      assert_equal( sha, sha_new,  "check file #{ file[:filename] }")
 
       # filename check
       assert_equal( file[:filename], attachments[0].filename )