Some performance improvements.

This commit is contained in:
Martin Edenhofer 2015-05-01 09:44:31 +02:00
parent 8c2ea52b3c
commit 1a72bfd98a

View file

@ -89,12 +89,18 @@ module Import::OTRS2
data['Key'] = Setting.get('import_otrs_endpoint_key') data['Key'] = Setting.get('import_otrs_endpoint_key')
log 'POST: ' + url log 'POST: ' + url
log 'PARAMS: ' + data.inspect log 'PARAMS: ' + data.inspect
open_timeout = 10
read_timeout = 120
if data.empty?
open_timeout = 6
read_timeout = 20
end
response = UserAgent.post( response = UserAgent.post(
url, url,
data, data,
{ {
open_timeout: 6, open_timeout: open_timeout,
read_timeout: 60, read_timeout: read_timeout,
user: Setting.get('import_otrs_user'), user: Setting.get('import_otrs_user'),
password: Setting.get('import_otrs_password'), password: Setting.get('import_otrs_password'),
}, },
@ -343,7 +349,7 @@ module Import::OTRS2
# create customers # create customers
count = 0 count = 0
steps = 30 steps = 50
run = true run = true
while run while run
count += steps count += steps
@ -360,6 +366,8 @@ module Import::OTRS2
thread_count = 8 thread_count = 8
threads = {} threads = {}
count = 0 count = 0
memory_clear_after_loops = 10
memory_clear_loop_counter = 0
locks = { User: {} } locks = { User: {} }
(1..thread_count).each {|thread| (1..thread_count).each {|thread|
threads[thread] = Thread.new { threads[thread] = Thread.new {
@ -369,6 +377,11 @@ module Import::OTRS2
run = true run = true
steps = 20 steps = 20
while run while run
memory_clear_loop_counter++
if memory_clear_loop_counter == memory_clear_after_loops
GC.start
memory_clear_loop_counter = 0
end
count += steps count += steps
log "loading... thread# #{thread} ..." log "loading... thread# #{thread} ..."
offset = count - steps offset = count - steps
@ -512,284 +525,287 @@ module Import::OTRS2
result.each {|record| result.each {|record|
# cleanup values ActiveRecord::Base.transaction do
_cleanup(record)
ticket_new = { # cleanup values
title: '', _cleanup(record)
created_by_id: 1,
updated_by_id: 1,
}
map[:Ticket].each { |key, value|
if record[key.to_s] && record[key.to_s].class == String
ticket_new[value] = Encode.conv( 'utf8', record[key.to_s] )
else
ticket_new[value] = record[key.to_s]
end
}
ticket_old = Ticket.where( id: ticket_new[:id] ).first
# find owner ticket_new = {
if ticket_new[:owner] title: '',
user = User.lookup( login: ticket_new[:owner].downcase )
if user
ticket_new[:owner_id] = user.id
else
ticket_new[:owner_id] = 1
end
ticket_new.delete(:owner)
end
# find customer
if ticket_new[:customer]
user = User.lookup( login: ticket_new[:customer].downcase )
if user
ticket_new[:customer_id] = user.id
else
ticket_new[:customer_id] = 1
end
ticket_new.delete(:customer)
else
ticket_new[:customer_id] = 1
end
# set state types
if ticket_old
log "update Ticket.find(#{ticket_new[:id]})"
ticket_old.update_attributes(ticket_new)
else
log "add Ticket.find(#{ticket_new[:id]})"
ticket = Ticket.new(ticket_new)
ticket.id = ticket_new[:id]
ticket.save
end
record['Articles'].each { |article|
# get article values
article_new = {
created_by_id: 1, created_by_id: 1,
updated_by_id: 1, updated_by_id: 1,
} }
map[:Article].each { |key, value| map[:Ticket].each { |key, value|
if article[key.to_s] if record[key.to_s] && record[key.to_s].class == String
article_new[value] = Encode.conv( 'utf8', article[key.to_s] ) ticket_new[value] = Encode.conv( 'utf8', record[key.to_s] )
else
ticket_new[value] = record[key.to_s]
end end
} }
ticket_old = Ticket.where( id: ticket_new[:id] ).first
# create customer/sender if needed # find owner
if article_new[:sender] == 'customer' && article_new[:created_by_id].to_i == 1 && !article_new[:from].empty? if ticket_new[:owner]
user = User.lookup( login: ticket_new[:owner].downcase )
email = nil if user
begin ticket_new[:owner_id] = user.id
email = Mail::Address.new( article_new[:from] ).address else
rescue ticket_new[:owner_id] = 1
email = article_new[:from]
if article_new[:from] =~ /<(.+?)>/
email = $1
end
end end
ticket_new.delete(:owner)
# create article user if not exists
while locks[:User][ email ]
log "user #{email} is locked"
sleep 1
end
# lock user
locks[:User][ email ] = true
user = User.where( email: email ).first
if !user
user = User.where( login: email ).first
end
if !user
begin
display_name = Mail::Address.new( article_new[:from] ).display_name ||
( Mail::Address.new( article_new[:from] ).comments && Mail::Address.new( article_new[:from] ).comments[0] )
rescue
display_name = article_new[:from]
end
# do extra decoding because we needed to use field.value
display_name = Mail::Field.new( 'X-From', display_name ).to_s
roles = Role.lookup( name: 'Customer' )
user = User.create(
login: email,
firstname: display_name,
lastname: '',
email: email,
password: '',
active: true,
role_ids: [roles.id],
updated_by_id: 1,
created_by_id: 1,
)
end
article_new[:created_by_id] = user.id
# unlock user
locks[:User][ email ] = false
end end
if article_new[:sender] == 'customer' # find customer
article_new[:sender_id] = Ticket::Article::Sender.lookup( name: 'Customer' ).id if ticket_new[:customer]
article_new.delete( :sender ) user = User.lookup( login: ticket_new[:customer].downcase )
end if user
if article_new[:sender] == 'agent' ticket_new[:customer_id] = user.id
article_new[:sender_id] = Ticket::Article::Sender.lookup( name: 'Agent' ).id else
article_new.delete( :sender ) ticket_new[:customer_id] = 1
end end
if article_new[:sender] == 'system' ticket_new.delete(:customer)
article_new[:sender_id] = Ticket::Article::Sender.lookup( name: 'System' ).id
article_new.delete( :sender )
end
if article_new[:type] == 'email-external'
article_new[:type_id] = Ticket::Article::Type.lookup( name: 'email' ).id
article_new[:internal] = false
elsif article_new[:type] == 'email-internal'
article_new[:type_id] = Ticket::Article::Type.lookup( name: 'email' ).id
article_new[:internal] = true
elsif article_new[:type] == 'note-external'
article_new[:type_id] = Ticket::Article::Type.lookup( name: 'note' ).id
article_new[:internal] = false
elsif article_new[:type] == 'note-internal'
article_new[:type_id] = Ticket::Article::Type.lookup( name: 'note' ).id
article_new[:internal] = true
elsif article_new[:type] == 'phone'
article_new[:type_id] = Ticket::Article::Type.lookup( name: 'phone' ).id
article_new[:internal] = false
elsif article_new[:type] == 'webrequest'
article_new[:type_id] = Ticket::Article::Type.lookup( name: 'web' ).id
article_new[:internal] = false
else else
article_new[:type_id] = 9 ticket_new[:customer_id] = 1
end end
article_new.delete( :type )
article_old = Ticket::Article.where( id: article_new[:id] ).first
# set state types # set state types
if article_old if ticket_old
log "update Ticket::Article.find(#{article_new[:id]})" log "update Ticket.find(#{ticket_new[:id]})"
article_old.update_attributes(article_new) ticket_old.update_attributes(ticket_new)
else else
log "add Ticket::Article.find(#{article_new[:id]})" log "add Ticket.find(#{ticket_new[:id]})"
article = Ticket::Article.new(article_new) ticket = Ticket.new(ticket_new)
article.id = article_new[:id] ticket.id = ticket_new[:id]
article.save ticket.save
end end
} record['Articles'].each { |article|
#puts "HS: #{record['History'].inspect}"
record['History'].each { |history| # get article values
if history['HistoryType'] == 'NewTicket' article_new = {
#puts "HS.add( #{history.inspect} )" created_by_id: 1,
res = History.add( updated_by_id: 1,
id: history['HistoryID'], }
o_id: history['TicketID'], map[:Article].each { |key, value|
history_type: 'created', if article[key.to_s]
history_object: 'Ticket', article_new[value] = Encode.conv( 'utf8', article[key.to_s] )
created_at: history['CreateTime'],
created_by_id: history['CreateBy']
)
#puts "res #{res.inspect}"
end
if history['HistoryType'] == 'StateUpdate'
data = history['Name']
# "%%new%%open%%"
from = nil
to = nil
if data =~ /%%(.+?)%%(.+?)%%/
from = $1
to = $2
state_from = Ticket::State.lookup( name: from )
state_to = Ticket::State.lookup( name: to )
if state_from
from_id = state_from.id
end end
if state_to }
to_id = state_to.id
# create customer/sender if needed
if article_new[:sender] == 'customer' && article_new[:created_by_id].to_i == 1 && !article_new[:from].empty?
email = nil
begin
email = Mail::Address.new( article_new[:from] ).address
rescue
email = article_new[:from]
if article_new[:from] =~ /<(.+?)>/
email = $1
end
end end
# create article user if not exists
while locks[:User][ email ]
log "user #{email} is locked"
sleep 1
end
# lock user
locks[:User][ email ] = true
user = User.where( email: email ).first
if !user
user = User.where( login: email ).first
end
if !user
begin
display_name = Mail::Address.new( article_new[:from] ).display_name ||
( Mail::Address.new( article_new[:from] ).comments && Mail::Address.new( article_new[:from] ).comments[0] )
rescue
display_name = article_new[:from]
end
# do extra decoding because we needed to use field.value
display_name = Mail::Field.new( 'X-From', display_name ).to_s
roles = Role.lookup( name: 'Customer' )
user = User.create(
login: email,
firstname: display_name,
lastname: '',
email: email,
password: '',
active: true,
role_ids: [roles.id],
updated_by_id: 1,
created_by_id: 1,
)
end
article_new[:created_by_id] = user.id
# unlock user
locks[:User][ email ] = false
end end
History.add(
id: history['HistoryID'], if article_new[:sender] == 'customer'
o_id: history['TicketID'], article_new[:sender_id] = Ticket::Article::Sender.lookup( name: 'Customer' ).id
history_type: 'updated', article_new.delete( :sender )
history_object: 'Ticket',
history_attribute: 'state',
value_from: from,
id_from: from_id,
value_to: to,
id_to: to_id,
created_at: history['CreateTime'],
created_by_id: history['CreateBy']
)
end
if history['HistoryType'] == 'Move'
data = history['Name']
# "%%Queue1%%5%%Postmaster%%1"
from = nil
to = nil
if data =~ /%%(.+?)%%(.+?)%%(.+?)%%(.+?)$/
from = $1
from_id = $2
to = $3
to_id = $4
end end
History.add( if article_new[:sender] == 'agent'
id: history['HistoryID'], article_new[:sender_id] = Ticket::Article::Sender.lookup( name: 'Agent' ).id
o_id: history['TicketID'], article_new.delete( :sender )
history_type: 'updated',
history_object: 'Ticket',
history_attribute: 'group',
value_from: from,
value_to: to,
id_from: from_id,
id_to: to_id,
created_at: history['CreateTime'],
created_by_id: history['CreateBy']
)
end
if history['HistoryType'] == 'PriorityUpdate'
data = history['Name']
# "%%3 normal%%3%%5 very high%%5"
from = nil
to = nil
if data =~ /%%(.+?)%%(.+?)%%(.+?)%%(.+?)$/
from = $1
from_id = $2
to = $3
to_id = $4
end end
History.add( if article_new[:sender] == 'system'
id: history['HistoryID'], article_new[:sender_id] = Ticket::Article::Sender.lookup( name: 'System' ).id
o_id: history['TicketID'], article_new.delete( :sender )
history_type: 'updated', end
history_object: 'Ticket',
history_attribute: 'priority', if article_new[:type] == 'email-external'
value_from: from, article_new[:type_id] = Ticket::Article::Type.lookup( name: 'email' ).id
value_to: to, article_new[:internal] = false
id_from: from_id, elsif article_new[:type] == 'email-internal'
id_to: to_id, article_new[:type_id] = Ticket::Article::Type.lookup( name: 'email' ).id
created_at: history['CreateTime'], article_new[:internal] = true
created_by_id: history['CreateBy'] elsif article_new[:type] == 'note-external'
) article_new[:type_id] = Ticket::Article::Type.lookup( name: 'note' ).id
end article_new[:internal] = false
if history['ArticleID'] && history['ArticleID'] != 0 elsif article_new[:type] == 'note-internal'
History.add( article_new[:type_id] = Ticket::Article::Type.lookup( name: 'note' ).id
id: history['HistoryID'], article_new[:internal] = true
o_id: history['ArticleID'], elsif article_new[:type] == 'phone'
history_type: 'created', article_new[:type_id] = Ticket::Article::Type.lookup( name: 'phone' ).id
history_object: 'Ticket::Article', article_new[:internal] = false
related_o_id: history['TicketID'], elsif article_new[:type] == 'webrequest'
related_history_object: 'Ticket', article_new[:type_id] = Ticket::Article::Type.lookup( name: 'web' ).id
created_at: history['CreateTime'], article_new[:internal] = false
created_by_id: history['CreateBy'] else
) article_new[:type_id] = 9
end end
} article_new.delete( :type )
article_old = Ticket::Article.where( id: article_new[:id] ).first
# set state types
if article_old
log "update Ticket::Article.find(#{article_new[:id]})"
article_old.update_attributes(article_new)
else
log "add Ticket::Article.find(#{article_new[:id]})"
article = Ticket::Article.new(article_new)
article.id = article_new[:id]
article.save
end
}
#puts "HS: #{record['History'].inspect}"
record['History'].each { |history|
if history['HistoryType'] == 'NewTicket'
#puts "HS.add( #{history.inspect} )"
res = History.add(
id: history['HistoryID'],
o_id: history['TicketID'],
history_type: 'created',
history_object: 'Ticket',
created_at: history['CreateTime'],
created_by_id: history['CreateBy']
)
#puts "res #{res.inspect}"
end
if history['HistoryType'] == 'StateUpdate'
data = history['Name']
# "%%new%%open%%"
from = nil
to = nil
if data =~ /%%(.+?)%%(.+?)%%/
from = $1
to = $2
state_from = Ticket::State.lookup( name: from )
state_to = Ticket::State.lookup( name: to )
if state_from
from_id = state_from.id
end
if state_to
to_id = state_to.id
end
end
History.add(
id: history['HistoryID'],
o_id: history['TicketID'],
history_type: 'updated',
history_object: 'Ticket',
history_attribute: 'state',
value_from: from,
id_from: from_id,
value_to: to,
id_to: to_id,
created_at: history['CreateTime'],
created_by_id: history['CreateBy']
)
end
if history['HistoryType'] == 'Move'
data = history['Name']
# "%%Queue1%%5%%Postmaster%%1"
from = nil
to = nil
if data =~ /%%(.+?)%%(.+?)%%(.+?)%%(.+?)$/
from = $1
from_id = $2
to = $3
to_id = $4
end
History.add(
id: history['HistoryID'],
o_id: history['TicketID'],
history_type: 'updated',
history_object: 'Ticket',
history_attribute: 'group',
value_from: from,
value_to: to,
id_from: from_id,
id_to: to_id,
created_at: history['CreateTime'],
created_by_id: history['CreateBy']
)
end
if history['HistoryType'] == 'PriorityUpdate'
data = history['Name']
# "%%3 normal%%3%%5 very high%%5"
from = nil
to = nil
if data =~ /%%(.+?)%%(.+?)%%(.+?)%%(.+?)$/
from = $1
from_id = $2
to = $3
to_id = $4
end
History.add(
id: history['HistoryID'],
o_id: history['TicketID'],
history_type: 'updated',
history_object: 'Ticket',
history_attribute: 'priority',
value_from: from,
value_to: to,
id_from: from_id,
id_to: to_id,
created_at: history['CreateTime'],
created_by_id: history['CreateBy']
)
end
if history['ArticleID'] && history['ArticleID'] != 0
History.add(
id: history['HistoryID'],
o_id: history['ArticleID'],
history_type: 'created',
history_object: 'Ticket::Article',
related_o_id: history['TicketID'],
related_history_object: 'Ticket',
created_at: history['CreateTime'],
created_by_id: history['CreateBy']
)
end
}
end
} }
end end