Improved storage backend.

This commit is contained in:
Martin Edenhofer 2014-04-28 09:44:36 +02:00
parent e1c1a95eb6
commit ec1f3afa73
11 changed files with 276 additions and 37 deletions

View file

@ -125,7 +125,7 @@ class TicketArticlesController < ApplicationController
# find file # find file
file = Store.find(params[:id]) file = Store.find(params[:id])
send_data( send_data(
file.store_file.data, file.content,
:filename => file.filename, :filename => file.filename,
:type => file.preferences['Content-Type'] || file.preferences['Mime-Type'], :type => file.preferences['Content-Type'] || file.preferences['Mime-Type'],
:disposition => 'inline' :disposition => 'inline'
@ -148,7 +148,7 @@ class TicketArticlesController < ApplicationController
if list if list
file = Store.find(list.first) file = Store.find(list.first)
send_data( send_data(
file.store_file.data, file.content,
:filename => file.filename, :filename => file.filename,
:type => 'message/rfc822', :type => 'message/rfc822',
:disposition => 'inline' :disposition => 'inline'

View file

@ -598,7 +598,7 @@ curl http://localhost/api/v1/users/image/8d6cca1c6bdc226cf2ba131e264ca2c7 -v -u
if list && list[0] if list && list[0]
file = Store.find( list[0] ) file = Store.find( list[0] )
send_data( send_data(
file.store_file.data, file.content,
:filename => file.filename, :filename => file.filename,
:type => file.preferences['Content-Type'] || file.preferences['Mime-Type'], :type => file.preferences['Content-Type'] || file.preferences['Mime-Type'],
:disposition => 'inline' :disposition => 'inline'

View file

@ -810,7 +810,7 @@ store attachments for this object
article_store.push Store.add( article_store.push Store.add(
:object => self.class.to_s, :object => self.class.to_s,
:o_id => self.id, :o_id => self.id,
:data => attachment.store_file.data, :data => attachment.content,
:filename => attachment.filename, :filename => attachment.filename,
:preferences => attachment.preferences, :preferences => attachment.preferences,
:created_by_id => self.created_by_id, :created_by_id => self.created_by_id,

View file

@ -41,7 +41,7 @@ class Channel::EmailBuild
mail.attachments[attachment.filename] = { mail.attachments[attachment.filename] = {
:content_type => attachment.preferences['Content-Type'], :content_type => attachment.preferences['Content-Type'],
:mime_type => attachment.preferences['Mime-Type'], :mime_type => attachment.preferences['Mime-Type'],
:content => attachment.store_file.data :content => attachment.content
} }
end end
end end

View file

@ -352,11 +352,10 @@ class Package < ApplicationModel
if !list || !list.first if !list || !list.first
raise "No such file in storage list #{name} #{version}" raise "No such file in storage list #{name} #{version}"
end end
store_file = list.first.store_file if !list.first.content
if !store_file
raise "No such file in storage #{name} #{version}" raise "No such file in storage #{name} #{version}"
end end
store_file.data list.first.content
end end
def self._read_file(file, fullpath = false) def self._read_file(file, fullpath = false)
@ -374,7 +373,7 @@ class Package < ApplicationModel
rescue => e rescue => e
raise 'ERROR: ' + e.inspect raise 'ERROR: ' + e.inspect
end end
return contents contents
end end
def self._write_file(file, permission, data) def self._write_file(file, permission, data)
@ -411,7 +410,7 @@ class Package < ApplicationModel
rescue => e rescue => e
raise 'ERROR: ' + e.inspect raise 'ERROR: ' + e.inspect
end end
return true true
end end
def self._delete_file(file, permission, data) def self._delete_file(file, permission, data)

View file

@ -8,6 +8,26 @@ class Store < ApplicationModel
belongs_to :store_file, :class_name => 'Store::File' belongs_to :store_file, :class_name => 'Store::File'
validates :filename, :presence => true validates :filename, :presence => true
=begin
add an attachment to storage
result = Store.add(
:object => 'Ticket::Article',
:o_id => 4711,
:data => binary_string,
:preferences => {
:content_type => 'image/png',
:content_id => 234,
}
)
returns
result = true
=end
def self.add(data) def self.add(data)
data = data.stringify_keys data = data.stringify_keys
@ -31,7 +51,7 @@ class Store < ApplicationModel
if file == nil if file == nil
file = Store::File.create( file = Store::File.create(
:data => data['data'], :data => data['data'],
:md5 => md5 :md5 => md5,
) )
end end
@ -44,9 +64,34 @@ class Store < ApplicationModel
# store meta data # store meta data
store = Store.create(data) store = Store.create(data)
return store true
end end
=begin
get attachment of object
list = Store.list(
:object => 'Ticket::Article',
:o_id => 4711,
)
returns
result = [store1, store2]
store1 = {
:size => 94123,
:filename => 'image.png',
:preferences => {
:content_type => 'image/png',
:content_id => 234,
}
}
store1.content # binary_string
=end
def self.list(data) def self.list(data)
# search # search
store_object_id = Store::Object.lookup( :name => data[:object] ) store_object_id = Store::Object.lookup( :name => data[:object] )
@ -55,6 +100,21 @@ class Store < ApplicationModel
return stores return stores
end end
=begin
remove an attachment to storage
result = Store.remove(
:object => 'Ticket::Article',
:o_id => 4711,
)
returns
result = true
=end
def self.remove(data) def self.remove(data)
# search # search
store_object_id = Store::Object.lookup( :name => data[:object] ) store_object_id = Store::Object.lookup( :name => data[:object] )
@ -62,22 +122,141 @@ class Store < ApplicationModel
where( :o_id => data[:o_id] ). where( :o_id => data[:o_id] ).
order('created_at ASC, id ASC') order('created_at ASC, id ASC')
stores.each do |store| stores.each do |store|
# check backend for references
files = Store.where( :store_file_id => store.store_file_id )
if files.count == 1 && files.first.id == store.id
Store::File.find( store.store_file_id ).destroy
end
store.destroy store.destroy
end end
return true return true
end end
class Object < ApplicationModel # get attachment
validates :name, :presence => true def content
file = Store::File.where( :id => self.store_file_id ).first
return if !file
if file.file_system
return file.read_from_fs
end
file.data
end
end
class Store::Object < ApplicationModel
validates :name, :presence => true
end
class Store::File < ApplicationModel
before_validation :add_md5
before_create :check_location
# generate file location
def get_locaton
# generate directory
base = Rails.root.to_s + "/storage/fs/"
path = self.md5.scan(/./).join('/')
location = "#{ base }/#{path}"
# create directory if not exists
if !File.exist?( location )
FileUtils.mkdir_p( location )
end
location += "/file"
end end
class File < ApplicationModel # read file from fs
before_validation :add_md5 def read_from_fs
puts "read from fs #{self.get_locaton}"
return if !File.exist?( self.get_locaton )
data = File.open( self.get_locaton, 'rb' )
content = data.read
private # check md5
def add_md5 md5 = Digest::MD5.hexdigest( content )
self.md5 = Digest::MD5.hexdigest( self.data ) if md5 != self.md5
raise "ERROR: Corrupt file in fs #{self.get_locaton}, md5 should be #{self.md5} but is #{md5}"
end
content
end
# write file to fs
def write_to_fs
# install file
permission = '600'
if !File.exist?( self.get_locaton )
puts "NOTICE: storge write '#{self.get_locaton}' (#{permission})"
file = File.new( self.get_locaton, 'wb' )
file.write( self.data )
file.close
end
File.chmod( permission.to_i(8), self.get_locaton )
# check md5
md5 = Digest::MD5.hexdigest( self.read_from_fs )
if md5 != self.md5
raise "ERROR: Corrupt file in fs #{self.get_locaton}, md5 should be #{self.md5} but is #{md5}"
end
true
end
# write file to db
def write_to_db
# read and check md5
content = self.read_from_fs
# store in database
self.data = content
self.save
# check md5 against db content
md5 = Digest::MD5.hexdigest( self.data )
if md5 != self.md5
raise "ERROR: Corrupt file in db #{self.get_locaton}, md5 should be #{self.md5} but is #{md5}"
end
true
end
def self.move_to_fs
Store::File.where( :file_system => false ).each {|item|
item.write_to_fs
item.update_attribute( :file_system, true )
item.update_attribute( :data, nil )
}
end
def self.move_to_db
Store::File.where( :file_system => true ).each {|item|
item.write_to_db
item.update_attribute( :file_system, false )
if File.exist?( item.get_locaton )
puts "NOTICE: storge remove '#{item.get_locaton}'"
File.delete( item.get_locaton )
end
}
end
private
def check_location
# write initial to fs if needed
if self.file_system && self.data
self.write_to_fs
self.data = nil
end end
end end
end def add_md5
if self.data && !self.md5
self.md5 = Digest::MD5.hexdigest( self.data )
end
end
end

View file

@ -67,7 +67,7 @@ returns
end end
data = { data = {
"_name" => attachment.filename, "_name" => attachment.filename,
"content" => Base64.encode64( attachment.store_file.data ) "content" => Base64.encode64( attachment.content )
} }
article_attributes['attachments'].push data article_attributes['attachments'].push data
} }

View file

@ -11,7 +11,7 @@ class CreateStorage < ActiveRecord::Migration
t.timestamps t.timestamps
end end
add_index :stores, [:store_object_id, :o_id] add_index :stores, [:store_object_id, :o_id]
create_table :store_objects do |t| create_table :store_objects do |t|
t.column :name, :string, :limit => 250, :null => false t.column :name, :string, :limit => 250, :null => false
t.column :note, :string, :limit => 250, :null => true t.column :note, :string, :limit => 250, :null => true
@ -20,8 +20,8 @@ class CreateStorage < ActiveRecord::Migration
add_index :store_objects, [:name], :unique => true add_index :store_objects, [:name], :unique => true
create_table :store_files do |t| create_table :store_files do |t|
t.column :data, :binary, :limit => 100.megabytes t.column :data, :binary, :limit => 200.megabytes, :null => true
t.column :md5, :string, :limit => 60, :null => false t.column :md5, :string, :limit => 60, :null => false
t.timestamps t.timestamps
end end
add_index :store_files, [:md5], :unique => true add_index :store_files, [:md5], :unique => true

View file

@ -1,6 +1,5 @@
class CreateActivityStream < ActiveRecord::Migration class CreateActivityStream < ActiveRecord::Migration
def up def up
create_table :activity_streams do |t| create_table :activity_streams do |t|
t.references :activity_stream_type, :null => false t.references :activity_stream_type, :null => false
t.references :activity_stream_object, :null => false t.references :activity_stream_object, :null => false

View file

@ -0,0 +1,9 @@
class UpdateStorage < ActiveRecord::Migration
def up
change_column :store_files, :data, :binary, :limit => 200.megabytes, :null => true
add_column :store_files, :file_system, :boolean, :null => false, :default => false
end
def down
end
end

View file

@ -1,32 +1,33 @@
# encoding: utf-8 # encoding: utf-8
require 'test_helper' require 'test_helper'
class StoreTest < ActiveSupport::TestCase class StoreTest < ActiveSupport::TestCase
test 'store attachment' do test 'store attachment' do
files = [ files = [
{ {
:data => 'hello world', :data => 'hello world',
:filename => 'test.txt', :filename => 'test.txt',
:o_id => 1,
}, },
{ {
:data => 'hello world äöüß', :data => 'hello world äöüß',
:filename => 'testäöüß.txt', :filename => 'testäöüß.txt',
:o_id => 2,
}, },
{ {
:data => IO.read('test/fixtures/test1.pdf'), :data => IO.read('test/fixtures/test1.pdf'),
:filename => 'test.pdf', :filename => 'test.pdf',
}, :o_id => 3,
},
] ]
files.each { |file| files.each { |file|
md5 = Digest::MD5.hexdigest( file[:data] ) md5 = Digest::MD5.hexdigest( file[:data] )
# add attachments # add attachments
store = Store.add( store = Store.add(
:object => 'Test', :object => 'Test',
:o_id => 1, :o_id => file[:o_id],
:data => file[:data], :data => file[:data],
:filename => file[:filename], :filename => file[:filename],
:preferences => {}, :preferences => {},
@ -37,24 +38,76 @@ class StoreTest < ActiveSupport::TestCase
# get list of attachments # get list of attachments
attachments = Store.list( attachments = Store.list(
:object => 'Test', :object => 'Test',
:o_id => 1 :o_id => file[:o_id],
) )
assert attachments assert attachments
# md5 check # md5 check
md5_new = Digest::MD5.hexdigest( attachments[0].store_file.data ) md5_new = Digest::MD5.hexdigest( attachments[0].content )
assert_equal( md5, md5_new ) assert_equal( md5, md5_new, "check file #{ file[:filename] }")
# filename check # filename check
assert_equal( file[:filename], attachments[0].filename ) assert_equal( file[:filename], attachments[0].filename )
# delete attachments }
Store::File.move_to_fs
files.each { |file|
md5 = Digest::MD5.hexdigest( file[:data] )
# get list of attachments
attachments = Store.list(
:object => 'Test',
:o_id => file[:o_id],
)
assert attachments
# md5 check
md5_new = Digest::MD5.hexdigest( attachments[0].content )
assert_equal( md5, md5_new, "check file #{ file[:filename] }")
# filename check
assert_equal( file[:filename], attachments[0].filename )
}
Store::File.move_to_db
files.each { |file|
md5 = Digest::MD5.hexdigest( file[:data] )
# get list of attachments
attachments = Store.list(
:object => 'Test',
:o_id => file[:o_id],
)
assert attachments
# md5 check
md5_new = Digest::MD5.hexdigest( attachments[0].content )
assert_equal( md5, md5_new, "check file #{ file[:filename] }")
# filename check
assert_equal( file[:filename], attachments[0].filename )
}
# delete attachments
files.each { |file|
success = Store.remove( success = Store.remove(
:object => 'Test', :object => 'Test',
:o_id => 1 :o_id => file[:o_id],
) )
assert success assert success
} }
# check attachments again
files.each { |file|
attachments = Store.list(
:object => 'Test',
:o_id => file[:o_id],
)
assert !attachments[0]
}
end end
end end