From 353b8e7ac3e5eb2476a3815e3eac5cb371adefd6 Mon Sep 17 00:00:00 2001 From: Martin Edenhofer Date: Tue, 4 Nov 2014 10:06:41 +0100 Subject: [PATCH] Improved importer. --- .../app/controllers/getting_started.js.coffee | 6 +- .../app/controllers/import.js.coffee | 2 +- .../app/controllers/import_otrs.js.coffee | 9 +- .../app/controllers/object_manager.js.coffee | 3 + app/controllers/getting_started_controller.rb | 4 +- app/controllers/import_otrs_controller.rb | 5 +- app/controllers/users_controller.rb | 12 +- lib/import/otrs2.rb | 640 +++++++++--------- 8 files changed, 328 insertions(+), 353 deletions(-) diff --git a/app/assets/javascripts/app/controllers/getting_started.js.coffee b/app/assets/javascripts/app/controllers/getting_started.js.coffee index 332fbb157..603878e32 100644 --- a/app/assets/javascripts/app/controllers/getting_started.js.coffee +++ b/app/assets/javascripts/app/controllers/getting_started.js.coffee @@ -32,7 +32,7 @@ class Index extends App.ControllerContent success: (data, status, xhr) => # redirect to login if master user already exists - if data.setup_done + if @Config.get('system_init_done') @navigate '#login' return @@ -87,7 +87,7 @@ class Base extends App.ControllerContent success: (data, status, xhr) => # redirect to login if master user already exists - if data.setup_done + if @Config.get('system_init_done') @navigate '#login' return @@ -305,7 +305,7 @@ class Admin extends App.ControllerContent success: (data, status, xhr) => # redirect to login if master user already exists - if data.setup_done + if @Config.get('system_init_done') @navigate '#login' return diff --git a/app/assets/javascripts/app/controllers/import.js.coffee b/app/assets/javascripts/app/controllers/import.js.coffee index fca9c66ed..4bc8cdeda 100644 --- a/app/assets/javascripts/app/controllers/import.js.coffee +++ b/app/assets/javascripts/app/controllers/import.js.coffee @@ -20,7 +20,7 @@ class Import extends App.ControllerContent success: (data, status, xhr) => # redirect to login if master user already exists - if data.setup_done + if @Config.get('system_init_done') @navigate '#login' return diff --git a/app/assets/javascripts/app/controllers/import_otrs.js.coffee b/app/assets/javascripts/app/controllers/import_otrs.js.coffee index 11afbd641..4ee161748 100644 --- a/app/assets/javascripts/app/controllers/import_otrs.js.coffee +++ b/app/assets/javascripts/app/controllers/import_otrs.js.coffee @@ -29,7 +29,7 @@ class Index extends App.ControllerContent success: (data, status, xhr) => # redirect to login if master user already exists - if data.setup_done + if @Config.get('system_init_done') @navigate '#login' return @@ -132,12 +132,15 @@ class Index extends App.ControllerContent element.find('.js-total').text(item.total) element.find('progress').attr('max', item.total ) element.find('progress').attr('value', item.done ) - if item.total is item.done + if item.total <= item.done element.addClass('is-done') else element.removeClass('is-done') - @delay( @updateMigration, 3000 ) +#js-finished +#@Config.set('system_init_done', true) + + @delay( @updateMigration, 5000 ) ) App.Config.set( 'import/otrs', Index, 'Routes' ) diff --git a/app/assets/javascripts/app/controllers/object_manager.js.coffee b/app/assets/javascripts/app/controllers/object_manager.js.coffee index 3f92805cd..496ab320c 100644 --- a/app/assets/javascripts/app/controllers/object_manager.js.coffee +++ b/app/assets/javascripts/app/controllers/object_manager.js.coffee @@ -2,6 +2,9 @@ class Index extends App.ControllerTabs constructor: -> super + # check authentication + return if !@authenticate() + # get data @ajax( id: 'object_manager_attributes_list', diff --git a/app/controllers/getting_started_controller.rb b/app/controllers/getting_started_controller.rb index a4e41fe1c..54ab174bf 100644 --- a/app/controllers/getting_started_controller.rb +++ b/app/controllers/getting_started_controller.rb @@ -331,7 +331,9 @@ curl http://localhost/api/v1/getting_started.json -v -u #{login}:#{password} return false end render :json => { - :setup_done => true, + :setup_done => true, + :import_mode => Setting.get('import_mode'), + :import_backend => Setting.get('import_backend'), } true end diff --git a/app/controllers/import_otrs_controller.rb b/app/controllers/import_otrs_controller.rb index 95af588e0..cc0bea4d0 100644 --- a/app/controllers/import_otrs_controller.rb +++ b/app/controllers/import_otrs_controller.rb @@ -48,9 +48,10 @@ class ImportOtrsController < ApplicationController url.gsub!(/([^:])(\/+\/)/, "\\1/") response = UserAgent.request( url ) + #Setting.set('import_mode', true) + Setting.set('import_backend', 'otrs') Setting.set('import_otrs_endpoint', url) Setting.set('import_otrs_endpoint_key', '01234567899876543210') - Setting.set('import_backend', 'otrs') if response.body =~ /zammad migrator/ render :json => { :url => url, @@ -74,7 +75,7 @@ class ImportOtrsController < ApplicationController return if setup_done_response Setting.set('import_mode', true) - welcome = Import::OTRS2.save_statisitic + welcome = Import::OTRS2.connection_test if !welcome render :json => { :message => 'Migrator can\'t read OTRS output!', diff --git a/app/controllers/users_controller.rb b/app/controllers/users_controller.rb index 7edb13d50..b291e4ce4 100644 --- a/app/controllers/users_controller.rb +++ b/app/controllers/users_controller.rb @@ -194,17 +194,9 @@ curl http://localhost/api/v1/users.json -v -u #{login}:#{password} -H "Content-T user.save - # if first user set init done + # if first user was added, set system init done if count <= 2 - Setting.create_or_update( - :title => 'System Init Done', - :name => 'system_init_done', - :area => 'Core', - :description => 'Defines if application is in init mode.', - :options => {}, - :state => true, - :frontend => true - ) + Setting.set( 'system_init_done', true ) end # send inviteation if needed / only if session exists diff --git a/lib/import/otrs2.rb b/lib/import/otrs2.rb index 667417db6..91f725c8d 100644 --- a/lib/import/otrs2.rb +++ b/lib/import/otrs2.rb @@ -2,7 +2,24 @@ module Import end module Import::OTRS2 - def self.request_json(part) +=begin + + result = request_json('Subaction=List', 1) + + return + + { some json structure } + + + result = request_json('Subaction=List') + + return + + "some data string" + +=end + + def self.request_json(part, data_only = false) response = request(part) if !response raise "Can't connect to Zammad Migrator" @@ -14,9 +31,25 @@ module Import::OTRS2 if !result raise "Invalid response" end - result + if data_only + result['Result'] + else + result + end end +=begin + + start get request to backend, add auth data automatically + + result = request('Subaction=List') + + return + + "some data string" + +=end + def self.request(part) url = Setting.get('import_otrs_endpoint') + part + ';Key=' + Setting.get('import_otrs_endpoint_key') puts 'GET: ' + url @@ -31,49 +64,20 @@ module Import::OTRS2 puts "ERROR: #{response.error}" return end - return response + response end - def self.connection_test - return self.request_json('') - end +=begin - def self.save_statisitic - statistic = self.request_json(';Subaction=List') - # save process - if statistic - Cache.write('import_otrs_stats', statistic) - end - statistic - end + start post request to backend, add auth data automatically - def self.get_statisitic - cache = Cache.get('import_otrs_stats') - if cache - return cache - end - self.save_statisitic - end + result = request('Subaction=List') - def self.get_current_state - total = self.get_statisitic - base = Group.count + Ticket::State.count + Ticket::Priority.count - data = { - :Base => { - :done => base, - :total => total['Base'] || 0, - }, - :User => { - :done => User.count, - :total => total['User'] || 0, - }, - :Ticket => { - :done => Ticket.count, - :total => total['Ticket'] || 0, - }, - } - data - end + return + + "some data string" + +=end def self.post(data) url = Setting.get('import_otrs_endpoint') @@ -92,17 +96,125 @@ module Import::OTRS2 puts "ERROR: #{response.error}" return end - return response + response end +=begin + + start post request to backend, add auth data automatically + + result = json('some response string') + + return + + {} + +=end + def self.json(response) data = Encode.conv( 'utf8', response.body.to_s ) JSON.parse( data ) end + def self.load( object, limit = '', offset = '' ) + request_json( ";Subaction=Export;Object=#{object};Limit=#{limit};Offset=#{offset}", 1 ) + end + +=begin + + start get request to backend to check connection + + result = connection_test + + return + + true | false + +=end + + def self.connection_test + return self.request_json('') + end + +=begin + + get object statisitic from server ans save it in cache + + result = get_statisitic('Subaction=List') + + return + + { + 'Ticket' => 1234, + 'User' => 123, + 'SomeObject' => 999, + } + +=end + + def self.get_statisitic + + # check cache + cache = Cache.get('import_otrs_stats') + if cache + return cache + end + + # retrive statistic + statistic = self.request_json(';Subaction=List', 1) + if statistic + Cache.write('import_otrs_stats', statistic) + end + statistic + end + +=begin + + return current import state + + result = get_current_state + + return + + { + :Ticket => { + :total => 1234, + :done => 13, + }, + :Base => { + :total => 1234, + :done => 13, + }, + } + +=end + + def self.get_current_state + data = self.get_statisitic + base = Group.count + Ticket::State.count + Ticket::Priority.count + base_total = data['Queue'] + data['State'] + data['Priority'] + user = User.count + user_total = data['User'] + data['CustomerUser'] + data = { + :Base => { + :done => base, + :total => base_total || 0, + }, + :User => { + :done => user, + :total => user_total || 0, + }, + :Ticket => { + :done => Ticket.count, + :total => data['Ticket'] || 0, + }, + } + data + end + # # start import - # + # # Import::OTRS2.start # @@ -122,97 +234,61 @@ module Import::OTRS2 "API key not valid!" end - # get objects to import - object_list = request_json(";Subaction=List") - #puts "r #{result.inspect}" - #if !result['Success'] - # "API key not valid!" - #end - object_list.each {|object, object_count| - puts "#{object} - #{object_count}" - relations = request_json(";Subaction=List;Object=#{object}") - relations.each {|relation, relation_count| - #puts "lll #{relation}/#{relation_count}" - records = request_json(";Subaction=Export;Object=#{object};Attribute=#{relation}") - #puts "--- #{recoords.inspect}" - if object == 'Ticket' - if relation == 'State' - state(records) - elsif relation == 'Priority' - priority(records) - elsif relation == 'Queue' - ticket_group(records) - elsif relation == 'CustomerUser' - #customer(records) - customer - elsif relation == 'User' - user(records) - end - end - } - if object == 'Ticket' - ticket_fetch - elsif object == 'TicketArticle' - ticket_article_fetch - end - } - - puts "aaaa #{result.inspect}" - return - -#self.ticket('156115') -#return # create states - state + records = load('State') + state(records) # create priorities - priority + records = load('Priority') + priority(records) # create groups - ticket_group + records = load('Queue') + ticket_group(records) # create agents - user + records = load('User') + user(records) # create customers -# customer - - result = JSON.parse( response.body ) - result = result.reverse + count = 0 + steps = 20 + run = true + while run + count += steps + records = load('CustomerUser', steps, count-steps) + if !records || !records[0] + puts "all customers imported." + run = false + next + end + customer(records) + end Thread.abort_on_exception = true thread_count = 4 threads = {} + count = 0 + locks = {} (1..thread_count).each {|thread| - - - threads[thread] = Thread.new { sleep thread * 3 puts "Started import thread# #{thread} ..." run = true - steps = 100 - count = 0 + steps = 20 while run - sleep 2 - puts "Count=#{count};Offset=#{count}" - result = request_json( ";Subaction=Export;Object=Ticket;Limit=#{steps};Offset=#{count}") #;Count=100;Offset=#{count}" ) - if !result - run = false - end - count = count + steps - run = false - end - - - while run - ticket_ids = result.pop(20) - if !ticket_ids.empty? - self.ticket(ticket_ids) - else + count += steps + #sleep 2 + #puts "Limit=#{steps};Offset=#{count}" + puts "loading... thread# #{thread} ..." + #result = request_json( ";Subaction=Export;Object=Ticket;Limit=#{steps};Offset=#{count-steps}", 1) #;Count=100;Offset=#{count}" ) + records = load( 'Ticket', steps, count-steps ) + if !records || !records[0] puts "... thread# #{thread}, no more work." run = false + next end + _ticket_result(records, locks) end } } @@ -220,6 +296,8 @@ module Import::OTRS2 threads[thread].join } + Setting.set( 'system_init_done', true ) + return end @@ -277,25 +355,7 @@ module Import::OTRS2 self._ticket_result(result) end - def self.ticket_fetch - done = false - count = 0 - steps = 100 - while done == false - sleep 2 - puts "Count=#{count};Offset=#{count}" - result = request_json( ";Subaction=Export;Object=Ticket;Limit=#{steps};Offset=#{count}") #;Count=100;Offset=#{count}" ) - return if !result - count = count + steps - if result.empty? - done = true - end - puts "aa #{result}" - _ticket_result(result) - end - end - - def self._ticket_result(result) + def self._ticket_result(result, locks) # puts result.inspect map = { :Ticket => { @@ -363,7 +423,8 @@ module Import::OTRS2 # check if state already exists ticket_old = Ticket.where( :id => ticket_new[:id] ).first #puts 'TICKET OLD ' + ticket_old.inspect - # find user + + # find owner if ticket_new[:owner] user = User.lookup( :login => ticket_new[:owner] ) if user @@ -373,6 +434,8 @@ module Import::OTRS2 end ticket_new.delete(:owner) end + + # find customer if ticket_new[:customer] user = User.lookup( :login => ticket_new[:customer] ) if user @@ -395,76 +458,8 @@ module Import::OTRS2 ticket.id = ticket_new[:id] ticket.save end - end - } - end - def self.ticket_article_fetch - done = false - count = 0 - steps = 100 - while done == false - sleep 2 - puts "Count=#{count};Offset=#{count}" - result = request_json( ";Subaction=Export;Object=TicketArticle;Limit=#{steps};Offset=#{count}") #;Count=100;Offset=#{count}" ) - return if !result - count = count + steps - if result.empty? - done = true - end - puts "aa #{result}" - _ticket_article_result(result) - end - end - - def self._ticket_article_result(record) - - - map = { - :Ticket => { - :Changed => :updated_at, - :Created => :created_at, - :CreateBy => :created_by_id, - :TicketNumber => :number, - :QueueID => :group_id, - :StateID => :state_id, - :PriorityID => :priority_id, - :Owner => :owner, - :CustomerUserID => :customer, - :Title => :title, - :TicketID => :id, - :FirstResponse => :first_response, -# :FirstResponseTimeDestinationDate => :first_response_escal_date, -# :FirstResponseInMin => :first_response_in_min, -# :FirstResponseDiffInMin => :first_response_diff_in_min, - :Closed => :close_time, -# :SoltutionTimeDestinationDate => :close_time_escal_date, -# :CloseTimeInMin => :close_time_in_min, -# :CloseTimeDiffInMin => :close_time_diff_in_min, - }, - :Article => { - :SenderType => :sender, - :ArticleType => :type, - :TicketID => :ticket_id, - :ArticleID => :id, - :Body => :body, - :From => :from, - :To => :to, - :Cc => :cc, - :Subject => :subject, - :InReplyTo => :in_reply_to, - :MessageID => :message_id, -# :ReplyTo => :reply_to, - :References => :references, - :Changed => :updated_at, - :Created => :created_at, - :ChangedBy => :updated_by_id, - :CreatedBy => :created_by_id, - }, - } - - - record.each { |article| + record['Articles'].each { |article| # get article values article_new = { @@ -492,6 +487,14 @@ module Import::OTRS2 user = User.where( :login => email ).first end if !user + + # create article user if not exists + while locks[ email ] + puts "user #{email} is locked" + sleep 1 + end + locks[ email ] = true + begin display_name = Mail::Address.new( article_new[:from] ).display_name || ( Mail::Address.new( article_new[:from] ).comments && Mail::Address.new( article_new[:from] ).comments[0] ) @@ -514,6 +517,7 @@ module Import::OTRS2 :updated_by_id => 1, :created_by_id => 1, ) + locks[ email ] = false end article_new[:created_by_id] = user.id end @@ -567,41 +571,19 @@ module Import::OTRS2 article.save end - if article['Attachments'] - article['Attachments'].each {|file| - headers_store = { - 'Content-Type' => file['ContentType'], - 'Content-Alternative' => file['ContentAlternative'], - 'Content-ID' => file['ContentID'], - } - Store.add( - :object => 'TicketArticle', - :o_id => article_new[:id], - :data => file['Content'], - :filename => file['Filename'], - :preferences => headers_store - ) - } - - puts "---- #{article['Attachments'].inspect}" - end } - end - def self._ticket_history_result(result) - - record['History'].each { |history| # puts '-------' # puts history.inspect if history['HistoryType'] == 'NewTicket' History.add( - :id => history['HistoryID'], - :o_id => history['TicketID'], - :history_type => 'created', - :history_object => 'Ticket', - :created_at => history['CreateTime'], - :created_by_id => history['CreateBy'] + :id => history['HistoryID'], + :o_id => history['TicketID'], + :history_type => 'created', + :history_object => 'Ticket', + :created_at => history['CreateTime'], + :created_by_id => history['CreateBy'] ) end if history['HistoryType'] == 'StateUpdate' @@ -699,8 +681,8 @@ module Import::OTRS2 ) end } - #end - #} + end + } end def self.state(records) @@ -757,6 +739,7 @@ module Import::OTRS2 end } end + def self.priority(records) map = { @@ -797,6 +780,7 @@ module Import::OTRS2 end } end + def self.ticket_group(records) map = { :ChangeTime => :updated_at, @@ -856,113 +840,103 @@ module Import::OTRS2 records.each { |user| # puts 'USER: ' + user.inspect - _set_valid(user) + _set_valid(user) - role = Role.lookup( :name => 'Agent' ) - # get new attributes - user_new = { - :created_by_id => 1, - :updated_by_id => 1, - :source => 'OTRS Import', - :role_ids => [ role.id ], - } - map.each { |key,value| - if user[key.to_s] - user_new[value] = user[key.to_s] - end - } - - # check if state already exists -# user_old = User.where( :login => user_new[:login] ).first - user_old = User.where( :id => user_new[:id] ).first - - # set state types - if user_old - puts "update User.find(#{user_new[:id]})" -# puts 'Update User' + user_new.inspect - user_new.delete( :role_ids ) - user_old.update_attributes(user_new) - else - puts "add User.find(#{user_new[:id]})" -# puts 'Add User' + user_new.inspect - user = User.new(user_new) - user.id = user_new[:id] - user.save - end - -# end - } - end - def self.customer - done = false - count = 0 - steps = 100 - while done == false - sleep 2 - puts "Count=#{count};Offset=#{count}" - result = request_json( ";Subaction=Export;Object=Ticket;Attribute=CustomerUser;Limit=#{steps};Offset=#{count}") #;Count=100;Offset=#{count}" ) - return if !result - count = count + steps - map = { - :ChangeTime => :updated_at, - :CreateTime => :created_at, - :CreateBy => :created_by_id, - :ChangeBy => :updated_by_id, - :ValidID => :active, - :UserComment => :note, - :UserEmail => :email, - :UserFirstname => :firstname, - :UserLastname => :lastname, - # :UserTitle => - :UserLogin => :login, - :UserPassword => :password, - :UserPhone => :phone, - :UserFax => :fax, - :UserMobile => :mobile, - :UserStreet => :street, - :UserZip => :zip, - :UserCity => :city, - :UserCountry => :country, - }; - - done = true - result.each { |user| - done = false - _set_valid(user) - - role = Role.lookup( :name => 'Customer' ) - - # get new attributes - user_new = { - :created_by_id => 1, - :updated_by_id => 1, - :source => 'OTRS Import', - :role_ids => [role.id], - } - map.each { |key,value| - if user[key.to_s] - user_new[value] = user[key.to_s] - end - } - - # check if state already exists - # user_old = User.where( :login => user_new[:login] ).first - user_old = User.where( :login => user_new[:login] ).first - - # set state types - if user_old - puts "update User.find(#{user_new[:id]})" -# puts 'Update User' + user_new.inspect - user_old.update_attributes(user_new) - else -# puts 'Add User' + user_new.inspect - puts "add User.find(#{user_new[:id]})" - user = User.new(user_new) - user.save + role = Role.lookup( :name => 'Agent' ) + # get new attributes + user_new = { + :created_by_id => 1, + :updated_by_id => 1, + :source => 'OTRS Import', + :role_ids => [ role.id ], + } + map.each { |key,value| + if user[key.to_s] + user_new[value] = user[key.to_s] end } - end + + if user_new[:password] + user_new[:password] = "{sha2}#{user_new[:password]}" + end + # check if state already exists +# user_old = User.where( :login => user_new[:login] ).first + user_old = User.where( :id => user_new[:id] ).first + + # set state types + if user_old + puts "update User.find(#{user_new[:id]})" +# puts 'Update User' + user_new.inspect + user_new.delete( :role_ids ) + user_old.update_attributes(user_new) + else + puts "add User.find(#{user_new[:id]})" +# puts 'Add User' + user_new.inspect + user = User.new(user_new) + user.id = user_new[:id] + user.save + end + } end + def self.customer(records) + map = { + :ChangeTime => :updated_at, + :CreateTime => :created_at, + :CreateBy => :created_by_id, + :ChangeBy => :updated_by_id, + :ValidID => :active, + :UserComment => :note, + :UserEmail => :email, + :UserFirstname => :firstname, + :UserLastname => :lastname, +# :UserTitle => + :UserLogin => :login, + :UserPassword => :password, + :UserPhone => :phone, + :UserFax => :fax, + :UserMobile => :mobile, + :UserStreet => :street, + :UserZip => :zip, + :UserCity => :city, + :UserCountry => :country, + }; + + records.each { |user| + _set_valid(user) +puts "CUser #{user.inspect}" + role = Role.lookup( :name => 'Customer' ) + + # get new attributes + user_new = { + :created_by_id => 1, + :updated_by_id => 1, + :source => 'OTRS Import', + :role_ids => [role.id], + } + map.each { |key,value| + if user[key.to_s] + user_new[value] = user[key.to_s] + end + } + + # check if state already exists +# user_old = User.where( :login => user_new[:login] ).first + user_old = User.where( :login => user_new[:login] ).first + + # set state types + if user_old + puts "update User.find(#{user_new[:id]})" +# puts 'Update User' + user_new.inspect + user_old.update_attributes(user_new) + else +# puts 'Add User' + user_new.inspect + puts "add User.find(#{user_new[:id]})" + user = User.new(user_new) + user.save + end + } + end + def self._set_valid(record) # map if record['ValidID'] == '3'