Improved storage backend.

This commit is contained in:
Martin Edenhofer 2014-04-28 09:44:36 +02:00
parent e1c1a95eb6
commit ec1f3afa73
11 changed files with 276 additions and 37 deletions

View file

@ -125,7 +125,7 @@ class TicketArticlesController < ApplicationController
# find file
file = Store.find(params[:id])
send_data(
file.store_file.data,
file.content,
:filename => file.filename,
:type => file.preferences['Content-Type'] || file.preferences['Mime-Type'],
:disposition => 'inline'
@ -148,7 +148,7 @@ class TicketArticlesController < ApplicationController
if list
file = Store.find(list.first)
send_data(
file.store_file.data,
file.content,
:filename => file.filename,
:type => 'message/rfc822',
:disposition => 'inline'

View file

@ -598,7 +598,7 @@ curl http://localhost/api/v1/users/image/8d6cca1c6bdc226cf2ba131e264ca2c7 -v -u
if list && list[0]
file = Store.find( list[0] )
send_data(
file.store_file.data,
file.content,
:filename => file.filename,
:type => file.preferences['Content-Type'] || file.preferences['Mime-Type'],
:disposition => 'inline'

View file

@ -810,7 +810,7 @@ store attachments for this object
article_store.push Store.add(
:object => self.class.to_s,
:o_id => self.id,
:data => attachment.store_file.data,
:data => attachment.content,
:filename => attachment.filename,
:preferences => attachment.preferences,
:created_by_id => self.created_by_id,

View file

@ -41,7 +41,7 @@ class Channel::EmailBuild
mail.attachments[attachment.filename] = {
:content_type => attachment.preferences['Content-Type'],
:mime_type => attachment.preferences['Mime-Type'],
:content => attachment.store_file.data
:content => attachment.content
}
end
end

View file

@ -352,11 +352,10 @@ class Package < ApplicationModel
if !list || !list.first
raise "No such file in storage list #{name} #{version}"
end
store_file = list.first.store_file
if !store_file
if !list.first.content
raise "No such file in storage #{name} #{version}"
end
store_file.data
list.first.content
end
def self._read_file(file, fullpath = false)
@ -374,7 +373,7 @@ class Package < ApplicationModel
rescue => e
raise 'ERROR: ' + e.inspect
end
return contents
contents
end
def self._write_file(file, permission, data)
@ -411,7 +410,7 @@ class Package < ApplicationModel
rescue => e
raise 'ERROR: ' + e.inspect
end
return true
true
end
def self._delete_file(file, permission, data)

View file

@ -8,6 +8,26 @@ class Store < ApplicationModel
belongs_to :store_file, :class_name => 'Store::File'
validates :filename, :presence => true
=begin
add an attachment to storage
result = Store.add(
:object => 'Ticket::Article',
:o_id => 4711,
:data => binary_string,
:preferences => {
:content_type => 'image/png',
:content_id => 234,
}
)
returns
result = true
=end
def self.add(data)
data = data.stringify_keys
@ -31,7 +51,7 @@ class Store < ApplicationModel
if file == nil
file = Store::File.create(
:data => data['data'],
:md5 => md5
:md5 => md5,
)
end
@ -44,9 +64,34 @@ class Store < ApplicationModel
# store meta data
store = Store.create(data)
return store
true
end
=begin
get attachment of object
list = Store.list(
:object => 'Ticket::Article',
:o_id => 4711,
)
returns
result = [store1, store2]
store1 = {
:size => 94123,
:filename => 'image.png',
:preferences => {
:content_type => 'image/png',
:content_id => 234,
}
}
store1.content # binary_string
=end
def self.list(data)
# search
store_object_id = Store::Object.lookup( :name => data[:object] )
@ -55,6 +100,21 @@ class Store < ApplicationModel
return stores
end
=begin
remove an attachment to storage
result = Store.remove(
:object => 'Ticket::Article',
:o_id => 4711,
)
returns
result = true
=end
def self.remove(data)
# search
store_object_id = Store::Object.lookup( :name => data[:object] )
@ -62,22 +122,141 @@ class Store < ApplicationModel
where( :o_id => data[:o_id] ).
order('created_at ASC, id ASC')
stores.each do |store|
# check backend for references
files = Store.where( :store_file_id => store.store_file_id )
if files.count == 1 && files.first.id == store.id
Store::File.find( store.store_file_id ).destroy
end
store.destroy
end
return true
end
class Object < ApplicationModel
validates :name, :presence => true
# get attachment
def content
file = Store::File.where( :id => self.store_file_id ).first
return if !file
if file.file_system
return file.read_from_fs
end
file.data
end
end
class Store::Object < ApplicationModel
validates :name, :presence => true
end
class Store::File < ApplicationModel
before_validation :add_md5
before_create :check_location
# generate file location
def get_locaton
# generate directory
base = Rails.root.to_s + "/storage/fs/"
path = self.md5.scan(/./).join('/')
location = "#{ base }/#{path}"
# create directory if not exists
if !File.exist?( location )
FileUtils.mkdir_p( location )
end
location += "/file"
end
class File < ApplicationModel
before_validation :add_md5
# read file from fs
def read_from_fs
puts "read from fs #{self.get_locaton}"
return if !File.exist?( self.get_locaton )
data = File.open( self.get_locaton, 'rb' )
content = data.read
private
def add_md5
self.md5 = Digest::MD5.hexdigest( self.data )
# check md5
md5 = Digest::MD5.hexdigest( content )
if md5 != self.md5
raise "ERROR: Corrupt file in fs #{self.get_locaton}, md5 should be #{self.md5} but is #{md5}"
end
content
end
# write file to fs
def write_to_fs
# install file
permission = '600'
if !File.exist?( self.get_locaton )
puts "NOTICE: storge write '#{self.get_locaton}' (#{permission})"
file = File.new( self.get_locaton, 'wb' )
file.write( self.data )
file.close
end
File.chmod( permission.to_i(8), self.get_locaton )
# check md5
md5 = Digest::MD5.hexdigest( self.read_from_fs )
if md5 != self.md5
raise "ERROR: Corrupt file in fs #{self.get_locaton}, md5 should be #{self.md5} but is #{md5}"
end
true
end
# write file to db
def write_to_db
# read and check md5
content = self.read_from_fs
# store in database
self.data = content
self.save
# check md5 against db content
md5 = Digest::MD5.hexdigest( self.data )
if md5 != self.md5
raise "ERROR: Corrupt file in db #{self.get_locaton}, md5 should be #{self.md5} but is #{md5}"
end
true
end
def self.move_to_fs
Store::File.where( :file_system => false ).each {|item|
item.write_to_fs
item.update_attribute( :file_system, true )
item.update_attribute( :data, nil )
}
end
def self.move_to_db
Store::File.where( :file_system => true ).each {|item|
item.write_to_db
item.update_attribute( :file_system, false )
if File.exist?( item.get_locaton )
puts "NOTICE: storge remove '#{item.get_locaton}'"
File.delete( item.get_locaton )
end
}
end
private
def check_location
# write initial to fs if needed
if self.file_system && self.data
self.write_to_fs
self.data = nil
end
end
def add_md5
if self.data && !self.md5
self.md5 = Digest::MD5.hexdigest( self.data )
end
end
end

View file

@ -67,7 +67,7 @@ returns
end
data = {
"_name" => attachment.filename,
"content" => Base64.encode64( attachment.store_file.data )
"content" => Base64.encode64( attachment.content )
}
article_attributes['attachments'].push data
}

View file

@ -20,8 +20,8 @@ class CreateStorage < ActiveRecord::Migration
add_index :store_objects, [:name], :unique => true
create_table :store_files do |t|
t.column :data, :binary, :limit => 100.megabytes
t.column :md5, :string, :limit => 60, :null => false
t.column :data, :binary, :limit => 200.megabytes, :null => true
t.column :md5, :string, :limit => 60, :null => false
t.timestamps
end
add_index :store_files, [:md5], :unique => true

View file

@ -1,6 +1,5 @@
class CreateActivityStream < ActiveRecord::Migration
def up
create_table :activity_streams do |t|
t.references :activity_stream_type, :null => false
t.references :activity_stream_object, :null => false

View file

@ -0,0 +1,9 @@
class UpdateStorage < ActiveRecord::Migration
def up
change_column :store_files, :data, :binary, :limit => 200.megabytes, :null => true
add_column :store_files, :file_system, :boolean, :null => false, :default => false
end
def down
end
end

View file

@ -7,26 +7,27 @@ class StoreTest < ActiveSupport::TestCase
{
:data => 'hello world',
:filename => 'test.txt',
:o_id => 1,
},
{
:data => 'hello world äöüß',
:filename => 'testäöüß.txt',
:o_id => 2,
},
{
:data => IO.read('test/fixtures/test1.pdf'),
:filename => 'test.pdf',
:o_id => 3,
},
]
files.each { |file|
md5 = Digest::MD5.hexdigest( file[:data] )
# add attachments
store = Store.add(
:object => 'Test',
:o_id => 1,
:o_id => file[:o_id],
:data => file[:data],
:filename => file[:filename],
:preferences => {},
@ -37,24 +38,76 @@ class StoreTest < ActiveSupport::TestCase
# get list of attachments
attachments = Store.list(
:object => 'Test',
:o_id => 1
:o_id => file[:o_id],
)
assert attachments
# md5 check
md5_new = Digest::MD5.hexdigest( attachments[0].store_file.data )
assert_equal( md5, md5_new )
md5_new = Digest::MD5.hexdigest( attachments[0].content )
assert_equal( md5, md5_new, "check file #{ file[:filename] }")
# filename check
assert_equal( file[:filename], attachments[0].filename )
# delete attachments
}
Store::File.move_to_fs
files.each { |file|
md5 = Digest::MD5.hexdigest( file[:data] )
# get list of attachments
attachments = Store.list(
:object => 'Test',
:o_id => file[:o_id],
)
assert attachments
# md5 check
md5_new = Digest::MD5.hexdigest( attachments[0].content )
assert_equal( md5, md5_new, "check file #{ file[:filename] }")
# filename check
assert_equal( file[:filename], attachments[0].filename )
}
Store::File.move_to_db
files.each { |file|
md5 = Digest::MD5.hexdigest( file[:data] )
# get list of attachments
attachments = Store.list(
:object => 'Test',
:o_id => file[:o_id],
)
assert attachments
# md5 check
md5_new = Digest::MD5.hexdigest( attachments[0].content )
assert_equal( md5, md5_new, "check file #{ file[:filename] }")
# filename check
assert_equal( file[:filename], attachments[0].filename )
}
# delete attachments
files.each { |file|
success = Store.remove(
:object => 'Test',
:o_id => 1
:o_id => file[:o_id],
)
assert success
}
# check attachments again
files.each { |file|
attachments = Store.list(
:object => 'Test',
:o_id => file[:o_id],
)
assert !attachments[0]
}
end
end