From 12c0ae1150c9081cc59303799415318cd9f894b1 Mon Sep 17 00:00:00 2001 From: Thorsten Eckel Date: Thu, 7 May 2015 13:57:19 +0200 Subject: [PATCH] Replaced old Import::OTRS with new Import::OTRS2. --- app/controllers/import_otrs_controller.rb | 6 +- db/migrate/20130305065226_scheduler_create.rb | 2 +- lib/auth/otrs.rb | 2 +- lib/import/otrs.rb | 1357 +++++++++++------ lib/import/otrs2.rb | 1301 ---------------- lib/sso/otrs.rb | 2 +- test/integration/otrs_import_test.rb | 2 +- 7 files changed, 921 insertions(+), 1751 deletions(-) delete mode 100644 lib/import/otrs2.rb diff --git a/app/controllers/import_otrs_controller.rb b/app/controllers/import_otrs_controller.rb index 8529008b4..d7bba4483 100644 --- a/app/controllers/import_otrs_controller.rb +++ b/app/controllers/import_otrs_controller.rb @@ -73,7 +73,7 @@ class ImportOtrsController < ApplicationController return if setup_done_response Setting.set('import_mode', true) - welcome = Import::OTRS2.connection_test + welcome = Import::OTRS.connection_test if !welcome render json: { message: 'Migrator can\'t read OTRS output!', @@ -83,7 +83,7 @@ class ImportOtrsController < ApplicationController end # start migration - Import::OTRS2.delay.start + Import::OTRS.delay.start render json: { result: 'ok', @@ -93,7 +93,7 @@ class ImportOtrsController < ApplicationController def import_status return if setup_done_response - state = Import::OTRS2.current_state + state = Import::OTRS.current_state render json: { data: state, diff --git a/db/migrate/20130305065226_scheduler_create.rb b/db/migrate/20130305065226_scheduler_create.rb index eae9d9fa4..5039f5d89 100644 --- a/db/migrate/20130305065226_scheduler_create.rb +++ b/db/migrate/20130305065226_scheduler_create.rb @@ -19,7 +19,7 @@ class SchedulerCreate < ActiveRecord::Migration add_index :schedulers, [:name], unique: true Scheduler.create_or_update( name: 'Import OTRS diff load', - method: 'Import::OTRS2.diff_worker', + method: 'Import::OTRS.diff_worker', period: 60 * 3, prio: 1, active: true, diff --git a/lib/auth/otrs.rb b/lib/auth/otrs.rb index 3dab3b7b1..1715789dc 100644 --- a/lib/auth/otrs.rb +++ b/lib/auth/otrs.rb @@ -11,7 +11,7 @@ module Auth::Otrs return false if endpoint == 'http://otrs_host/otrs' # connect to OTRS - result = Import::OTRS2.auth( username, password ) + result = Import::OTRS.auth( username, password ) return false if !result return false if !result['groups_ro'] return false if !result['groups_rw'] diff --git a/lib/import/otrs.rb b/lib/import/otrs.rb index 81f91ea08..d8922da58 100644 --- a/lib/import/otrs.rb +++ b/lib/import/otrs.rb @@ -1,49 +1,150 @@ module Import end module Import::OTRS + +=begin + + result = request_json( :Subaction => 'List', 1) + + return + + { some json structure } + + result = request_json( :Subaction => 'List' ) + + return + + "some data string" + +=end + + def self.request_json(data, data_only = false) + response = post(data) + if !response + fail "Can't connect to Zammad Migrator" + end + if !response.success? + fail "Can't connect to Zammad Migrator" + end + result = json(response) + if !result + fail 'Invalid response' + end + if data_only + result['Result'] + else + result + end + end + +=begin + + start get request to backend, add auth data automatically + + result = request('Subaction=List') + + return + + "some data string" + +=end + def self.request(part) - url = Setting.get('import_otrs_endpoint') + '/' + part + ';Key=' + Setting.get('import_otrs_endpoint_key') - Rails.logger.info 'GET: ' + url - response = UserAgent.request( + url = Setting.get('import_otrs_endpoint') + part + ';Key=' + Setting.get('import_otrs_endpoint_key') + log 'GET: ' + url + response = UserAgent.get( url, + {}, { + open_timeout: 10, + read_timeout: 60, user: Setting.get('import_otrs_user'), password: Setting.get('import_otrs_password'), }, ) if !response.success? - Rails.logger.info "ERROR: #{response.error}" + log "ERROR: #{response.error}" return end response end - def self.post(base, data) - url = Setting.get('import_otrs_endpoint') + '/' + base + +=begin + + start post request to backend, add auth data automatically + + result = request('Subaction=List') + + return + + "some data string" + +=end + + def self.post(data, url = nil) + if !url + url = Setting.get('import_otrs_endpoint') + data['Action'] = 'ZammadMigrator' + end data['Key'] = Setting.get('import_otrs_endpoint_key') - Rails.logger.info 'POST: ' + url - response = UserAgent.request( + log 'POST: ' + url + log 'PARAMS: ' + data.inspect + open_timeout = 10 + read_timeout = 120 + if data.empty? + open_timeout = 6 + read_timeout = 20 + end + response = UserAgent.post( url, + data, { - method: 'post', - data: data, + open_timeout: open_timeout, + read_timeout: read_timeout, user: Setting.get('import_otrs_user'), password: Setting.get('import_otrs_password'), }, ) if !response.success? - Rails.logger.info "ERROR: #{response.error}" + log "ERROR: #{response.error}" return end response end +=begin + + start post request to backend, add auth data automatically + + result = json('some response string') + + return + + {} + +=end + def self.json(response) data = Encode.conv( 'utf8', response.body.to_s ) JSON.parse( data ) end +=begin + + start auth on OTRS - just for experimental reasons + + result = auth(username, password) + + return + + { ..user structure.. } + +=end + def self.auth(username, password) - response = post( 'public.pl', { Action: 'Export', Type: 'Auth', User: username, Pw: password } ) + url = Setting.get('import_otrs_endpoint') + url.gsub!('ZammadMigrator', 'ZammadSSO') + response = post( { Action: 'ZammadSSO', Subaction: 'Auth', User: username, Pw: password }, url ) return if !response return if !response.success? @@ -51,122 +152,264 @@ module Import::OTRS result end +=begin + + request session data - just for experimental reasons + + result = session(session_id) + + return + + { ..session structure.. } + +=end + def self.session(session_id) - response = post( 'public.pl', { Action: 'Export', Type: 'SessionCheck', SessionID: session_id } ) + url = Setting.get('import_otrs_endpoint') + url.gsub!('ZammadMigrator', 'ZammadSSO') + response = post( { Action: 'ZammadSSO', Subaction: 'SessionCheck', SessionID: session_id }, url ) return if !response return if !response.success? - result = json(response) result end - def self.permission_sync(user, result, config) +=begin - # check if required OTRS group exists - types = { - required_group_ro: 'groups_ro', - required_group_rw: 'groups_rw', - } - types.each {|config_key, result_key| - if config[config_key] - return false if !result[result_key].value?( config[config_key] ) - end - } + load objects from otrs - # sync roles / groups - if config[:group_ro_role_map] || config[:group_rw_role_map] - user.role_ids = [] - user.save - end - types = { - group_ro_role_map: 'groups_ro', - group_rw_role_map: 'groups_rw', - } - types.each {|config_key, result_key| - next if !config[config_key] - config[config_key].each {|otrs_group, role| - next if !result[result_key].value?( otrs_group ) - role_ids = user.role_ids - role = Role.where( name: role ).first - next if !role - role_ids.push role.id - user.role_ids = role_ids - user.save - } - } + result = load('SysConfig') - if config[:always_role] - config[:always_role].each {|role, active| - next if !active - role_ids = user.role_ids - role = Role.where( name: role ).first - next if !role - role_ids.push role.id - user.role_ids = role_ids - user.save - } - end + return + [ + { ..object1.. }, + { ..object2.. }, + { ..object3.. }, + ] + +=end + + def self.load( object, limit = '', offset = '', diff = 0 ) + request_json( { Subaction: 'Export', Object: object, Limit: limit, Offset: offset, Diff: diff }, 1 ) end - def self.start - Rails.logger.info 'Start import...' +=begin -# # set system in import mode -# Setting.set('import_mode', true) + start get request to backend to check connection + + result = connection_test + + return + + true | false + +=end + + def self.connection_test + self.request_json({}) + end + +=begin + + get object statistic from server ans save it in cache + + result = statistic('Subaction=List') + + return + + { + 'Ticket' => 1234, + 'User' => 123, + 'SomeObject' => 999, + } + +=end + + def self.statistic + + # check cache + cache = Cache.get('import_otrs_stats') + if cache + return cache + end + + # retrive statistic + statistic = self.request_json( { Subaction: 'List' }, 1) + if statistic + Cache.write('import_otrs_stats', statistic) + end + statistic + end + +=begin + + return current import state + + result = current_state + + return + + { + :Ticket => { + :total => 1234, + :done => 13, + }, + :Base => { + :total => 1234, + :done => 13, + }, + } + +=end + + def self.current_state + data = self.statistic + base = Group.count + Ticket::State.count + Ticket::Priority.count + base_total = data['Queue'] + data['State'] + data['Priority'] + user = User.count + user_total = data['User'] + data['CustomerUser'] + data = { + Base: { + done: base, + total: base_total || 0, + }, + User: { + done: user, + total: user_total || 0, + }, + Ticket: { + done: Ticket.count, + total: data['Ticket'] || 0, + }, + } + data + end + + # + # start import + # + # Import::OTRS.start + # + + def self.start + log 'Start import...' # check if system is in import mode if !Setting.get('import_mode') fail 'System is not in import mode!' end - response = request('public.pl?Action=Export') - return if !response - return if !response.success? + result = request_json({}) + if !result['Success'] + 'API key not valid!' + end + + # set settings + settings = load('SysConfig') + setting(settings) + + # dynamic fields + dynamic_fields = load('DynamicField') + #settings(dynamic_fields, settings) + + # email accounts + #accounts = load('PostMasterAccount') + #account(accounts) + + # email filter + #filters = load('PostMasterFilter') + #filter(filters) -#self.ticket('156115') -#return # create states - state + states = load('State') + ActiveRecord::Base.transaction do + state(states) + end # create priorities - priority + priorities = load('Priority') + ActiveRecord::Base.transaction do + priority(priorities) + end # create groups - ticket_group + queues = load('Queue') + ActiveRecord::Base.transaction do + ticket_group(queues) + end + + # get agents groups + groups = load('Group') + + # get agents roles + roles = load('Role') # create agents - user + users = load('User') + ActiveRecord::Base.transaction do + user(users, groups, roles, queues) + end + + # create organizations + organizations = load('Customer') + ActiveRecord::Base.transaction do + organization(organizations) + end # create customers -# customer - - result = JSON.parse( response.body ) - result = result.reverse + count = 0 + steps = 50 + run = true + while run + count += steps + records = load('CustomerUser', steps, count - steps) + if !records || !records[0] + log 'all customers imported.' + run = false + next + end + customer(records, organizations) + end Thread.abort_on_exception = true - thread_count = 4 - threads = {} + thread_count = 8 + threads = {} + count = 0 + locks = { User: {} } (1..thread_count).each {|thread| threads[thread] = Thread.new { + Thread.current[:thread_no] = thread sleep thread * 3 - Rails.logger.info "Started import thread# #{thread} ..." + log "Started import thread# #{thread} ..." run = true + steps = 20 while run - ticket_ids = result.pop(20) - if !ticket_ids.empty? - self.ticket(ticket_ids) - else - Rails.logger.info "... thread# #{thread}, no more work." - run = false + count += steps + log "loading... thread# #{thread} ..." + offset = count - steps + if offset != 0 + offset = count - steps + 1 end + records = load( 'Ticket', steps, count - steps) + if !records || !records[0] + log "... thread# #{thread}, no more work." + run = false + next + end + _ticket_result(records, locks, thread) end + ActiveRecord::Base.connection.close } } (1..thread_count).each {|thread| threads[thread].join } + Setting.set( 'system_init_done', true ) + #Setting.set( 'import_mode', false ) + + true end def self.diff_worker @@ -176,7 +419,7 @@ module Import::OTRS end def self.diff - Rails.logger.info 'Start diff...' + log 'Start diff...' # check if system is in import mode if !Setting.get('import_mode') @@ -184,45 +427,61 @@ module Import::OTRS end # create states - state + states = load('State') + state(states) # create priorities - priority + priorities = load('Priority') + priority(priorities) # create groups - ticket_group + queues = load('Queue') + ticket_group(queues) + + # get agents groups + groups = load('Group') + + # get agents roles + roles = load('Role') # create agents - user + users = load('User') + user(users, groups, roles, queues) - self.ticket_diff() + # create organizations + organizations = load('Customer') + organization(organizations) + + # get changed tickets + self.ticket_diff end - def self.ticket_diff() - url = 'public.pl?Action=Export;Type=TicketDiff;Limit=30' - response = request( url ) - return if !response - return if !response.success? - result = json(response) - self._ticket_result(result) + def self.ticket_diff + count = 0 + run = true + steps = 20 + locks = { User: {} } + while run + count += steps + log 'loading... diff ...' + offset = count - steps + if offset != 0 + offset = count - steps + 1 + end + records = load( 'Ticket', steps, count - steps, 1 ) + if !records || !records[0] + log '... no more work.' + run = false + next + end + _ticket_result(records, locks) + end + end - def self.ticket(ticket_ids) - url = 'public.pl?Action=Export;Type=Ticket;' - ticket_ids.each {|ticket_id| - url = url + "TicketID=#{CGI.escape ticket_id};" - } - response = request( url ) - return if !response - return if !response.success? - - result = json(response) - self._ticket_result(result) - end - - def self._ticket_result(result) -# Rails.logger.info result.inspect + def self._ticket_result(result, locks, _thread = '-') +# puts result.inspect map = { Ticket: { Changed: :updated_at, @@ -268,60 +527,68 @@ module Import::OTRS result.each {|record| - # use transaction + # cleanup values + _cleanup(record) + + _utf8_encode(record) + + ticket_new = { + title: '', + created_by_id: 1, + updated_by_id: 1, + } + map[:Ticket].each { |key, value| + next if !record.key?(key.to_s) + ticket_new[value] = record[key.to_s] + } + ticket_old = Ticket.where( id: ticket_new[:id] ).first + + # find owner + if ticket_new[:owner] + user = User.lookup( login: ticket_new[:owner].downcase ) + if user + ticket_new[:owner_id] = user.id + else + ticket_new[:owner_id] = 1 + end + ticket_new.delete(:owner) + end + + # find customer + if ticket_new[:customer] + user = User.lookup( login: ticket_new[:customer].downcase ) + if user + ticket_new[:customer_id] = user.id + else + ticket_new[:customer_id] = 1 + end + ticket_new.delete(:customer) + else + ticket_new[:customer_id] = 1 + end + + # set state types + if ticket_old + log "update Ticket.find(#{ticket_new[:id]})" + ticket_old.update_attributes(ticket_new) + else + log "add Ticket.find(#{ticket_new[:id]})" + ticket = Ticket.new(ticket_new) + ticket.id = ticket_new[:id] + ticket.save + end + + # utf8 encode + record['Articles'].each { |article| + _utf8_encode(article) + } + + # lookup customers to create first + record['Articles'].each { |article| + _article_based_customers(article, locks) + } + ActiveRecord::Base.transaction do - - ticket_new = { - title: '', - created_by_id: 1, - updated_by_id: 1, - } - map[:Ticket].each { |key, value| - if record['Ticket'][key.to_s] && record['Ticket'][key.to_s].class == String - ticket_new[value] = Encode.conv( 'utf8', record['Ticket'][key.to_s] ) - else - ticket_new[value] = record['Ticket'][key.to_s] - end - } -# Rails.logger.info key.to_s -# Rails.logger.info value.to_s -#Rails.logger.info 'new ticket data ' + ticket_new.inspect - # check if state already exists - ticket_old = Ticket.where( id: ticket_new[:id] ).first -#Rails.logger.info 'TICKET OLD ' + ticket_old.inspect - # find user - if ticket_new[:owner] - user = User.lookup( login: ticket_new[:owner] ) - if user - ticket_new[:owner_id] = user.id - else - ticket_new[:owner_id] = 1 - end - ticket_new.delete(:owner) - end - if ticket_new[:customer] - user = User.lookup( login: ticket_new[:customer] ) - if user - ticket_new[:customer_id] = user.id - else - ticket_new[:customer_id] = 1 - end - ticket_new.delete(:customer) - else - ticket_new[:customer_id] = 1 - end -# Rails.logger.info 'ttt' + ticket_new.inspect - # set state types - if ticket_old - Rails.logger.info "update Ticket.find(#{ticket_new[:id]})" - ticket_old.update_attributes(ticket_new) - else - Rails.logger.info "add Ticket.find(#{ticket_new[:id]})" - ticket = Ticket.new(ticket_new) - ticket.id = ticket_new[:id] - ticket.save - end - record['Articles'].each { |article| # get article values @@ -329,52 +596,11 @@ module Import::OTRS created_by_id: 1, updated_by_id: 1, } + map[:Article].each { |key, value| - if article[key.to_s] - article_new[value] = Encode.conv( 'utf8', article[key.to_s] ) - end + next if !article.key?(key.to_s) + article_new[value] = article[key.to_s] } - # create customer/sender if needed - if article_new[:sender] == 'customer' && article_new[:created_by_id].to_i == 1 && !article_new[:from].empty? - # set extra headers - begin - email = Mail::Address.new( article_new[:from] ).address - rescue - email = article_new[:from] - if article_new[:from] =~ /<(.+?)>/ - email = $1 - end - end - user = User.where( email: email ).first - if !user - user = User.where( login: email ).first - end - if !user - begin - display_name = Mail::Address.new( article_new[:from] ).display_name || - ( Mail::Address.new( article_new[:from] ).comments && Mail::Address.new( article_new[:from] ).comments[0] ) - rescue - display_name = article_new[:from] - end - - # do extra decoding because we needed to use field.value - display_name = Mail::Field.new( 'X-From', display_name ).to_s - - roles = Role.lookup( name: 'Customer' ) - user = User.create( - login: email, - firstname: display_name, - lastname: '', - email: email, - password: '', - active: true, - role_ids: [roles.id], - updated_by_id: 1, - created_by_id: 1, - ) - end - article_new[:created_by_id] = user.id - end if article_new[:sender] == 'customer' article_new[:sender_id] = Ticket::Article::Sender.lookup( name: 'Customer' ).id @@ -412,142 +638,135 @@ module Import::OTRS end article_new.delete( :type ) article_old = Ticket::Article.where( id: article_new[:id] ).first - #Rails.logger.info 'ARTICLE OLD ' + article_old.inspect + # set state types if article_old - Rails.logger.info "update Ticket::Article.find(#{article_new[:id]})" - # Rails.logger.info article_new.inspect + log "update Ticket::Article.find(#{article_new[:id]})" article_old.update_attributes(article_new) else - Rails.logger.info "add Ticket::Article.find(#{article_new[:id]})" + log "add Ticket::Article.find(#{article_new[:id]})" article = Ticket::Article.new(article_new) article.id = article_new[:id] article.save end - } + end - record['History'].each { |history| - # Rails.logger.info '-------' - # Rails.logger.info history.inspect - if history['HistoryType'] == 'NewTicket' - History.add( - id: history['HistoryID'], - o_id: history['TicketID'], - history_type: 'created', - history_object: 'Ticket', - created_at: history['CreateTime'], - created_by_id: history['CreateBy'] - ) - end - if history['HistoryType'] == 'StateUpdate' - data = history['Name'] - # "%%new%%open%%" - from = nil - to = nil - if data =~ /%%(.+?)%%(.+?)%%/ - from = $1 - to = $2 - state_from = Ticket::State.lookup( name: from ) - state_to = Ticket::State.lookup( name: to ) - if state_from - from_id = state_from.id - end - if state_to - to_id = state_to.id - end - end - # Rails.logger.info "STATE UPDATE (#{history['HistoryID']}): -> #{from}->#{to}" - History.add( - id: history['HistoryID'], - o_id: history['TicketID'], - history_type: 'updated', - history_object: 'Ticket', - history_attribute: 'state', - value_from: from, - id_from: from_id, - value_to: to, - id_to: to_id, - created_at: history['CreateTime'], - created_by_id: history['CreateBy'] - ) - end - if history['HistoryType'] == 'Move' - data = history['Name'] - # "%%Queue1%%5%%Postmaster%%1" - from = nil - to = nil - if data =~ /%%(.+?)%%(.+?)%%(.+?)%%(.+?)$/ - from = $1 - from_id = $2 - to = $3 - to_id = $4 - end - History.add( - id: history['HistoryID'], - o_id: history['TicketID'], - history_type: 'updated', - history_object: 'Ticket', - history_attribute: 'group', - value_from: from, - value_to: to, - id_from: from_id, - id_to: to_id, - created_at: history['CreateTime'], - created_by_id: history['CreateBy'] - ) - end - if history['HistoryType'] == 'PriorityUpdate' - data = history['Name'] - # "%%3 normal%%3%%5 very high%%5" - from = nil - to = nil - if data =~ /%%(.+?)%%(.+?)%%(.+?)%%(.+?)$/ - from = $1 - from_id = $2 - to = $3 - to_id = $4 - end - History.add( - id: history['HistoryID'], - o_id: history['TicketID'], - history_type: 'updated', - history_object: 'Ticket', - history_attribute: 'priority', - value_from: from, - value_to: to, - id_from: from_id, - id_to: to_id, - created_at: history['CreateTime'], - created_by_id: history['CreateBy'] - ) - end - - next if !history['ArticleID'] - next if history['ArticleID'] == 0 - - History.add( + #puts "HS: #{record['History'].inspect}" + record['History'].each { |history| + if history['HistoryType'] == 'NewTicket' + #puts "HS.add( #{history.inspect} )" + res = History.add( id: history['HistoryID'], - o_id: history['ArticleID'], + o_id: history['TicketID'], history_type: 'created', - history_object: 'Ticket::Article', - related_o_id: history['TicketID'], - related_history_object: 'Ticket', + history_object: 'Ticket', created_at: history['CreateTime'], created_by_id: history['CreateBy'] ) - } - end + #puts "res #{res.inspect}" + end + if history['HistoryType'] == 'StateUpdate' + data = history['Name'] + # "%%new%%open%%" + from = nil + to = nil + if data =~ /%%(.+?)%%(.+?)%%/ + from = $1 + to = $2 + state_from = Ticket::State.lookup( name: from ) + state_to = Ticket::State.lookup( name: to ) + if state_from + from_id = state_from.id + end + if state_to + to_id = state_to.id + end + end + History.add( + id: history['HistoryID'], + o_id: history['TicketID'], + history_type: 'updated', + history_object: 'Ticket', + history_attribute: 'state', + value_from: from, + id_from: from_id, + value_to: to, + id_to: to_id, + created_at: history['CreateTime'], + created_by_id: history['CreateBy'] + ) + end + if history['HistoryType'] == 'Move' + data = history['Name'] + # "%%Queue1%%5%%Postmaster%%1" + from = nil + to = nil + if data =~ /%%(.+?)%%(.+?)%%(.+?)%%(.+?)$/ + from = $1 + from_id = $2 + to = $3 + to_id = $4 + end + History.add( + id: history['HistoryID'], + o_id: history['TicketID'], + history_type: 'updated', + history_object: 'Ticket', + history_attribute: 'group', + value_from: from, + value_to: to, + id_from: from_id, + id_to: to_id, + created_at: history['CreateTime'], + created_by_id: history['CreateBy'] + ) + end + if history['HistoryType'] == 'PriorityUpdate' + data = history['Name'] + # "%%3 normal%%3%%5 very high%%5" + from = nil + to = nil + if data =~ /%%(.+?)%%(.+?)%%(.+?)%%(.+?)$/ + from = $1 + from_id = $2 + to = $3 + to_id = $4 + end + History.add( + id: history['HistoryID'], + o_id: history['TicketID'], + history_type: 'updated', + history_object: 'Ticket', + history_attribute: 'priority', + value_from: from, + value_to: to, + id_from: from_id, + id_to: to_id, + created_at: history['CreateTime'], + created_by_id: history['CreateBy'] + ) + end + + next if !history['ArticleID'] + next if history['ArticleID'] == 0 + + History.add( + id: history['HistoryID'], + o_id: history['ArticleID'], + history_type: 'created', + history_object: 'Ticket::Article', + related_o_id: history['TicketID'], + related_history_object: 'Ticket', + created_at: history['CreateTime'], + created_by_id: history['CreateBy'] + ) + } } end - def self.state - response = request( 'public.pl?Action=Export;Type=State' ) - return if !response - return if !response.success? - - result = json(response) -# Rails.logger.info result.inspect + # sync ticket states + def self.state(records) map = { ChangeTime: :updated_at, CreateTime: :created_at, @@ -559,12 +778,13 @@ module Import::OTRS Comment: :note, } + # rename states to handle not uniq issues Ticket::State.all.each {|state| state.name = state.name + '_tmp' state.save } - result.each { |state| + records.each { |state| _set_valid(state) # get new attributes @@ -573,14 +793,12 @@ module Import::OTRS updated_by_id: 1, } map.each { |key, value| - if state[key.to_s] - state_new[value] = state[key.to_s] - end + next if !state.key?(key.to_s) + state_new[value] = state[key.to_s] } # check if state already exists state_old = Ticket::State.where( id: state_new[:id] ).first -# Rails.logger.info 'st: ' + state['TypeName'] # set state types if state['TypeName'] == 'pending auto' @@ -589,7 +807,6 @@ module Import::OTRS state_type = Ticket::StateType.where( name: state['TypeName'] ).first state_new[:state_type_id] = state_type.id if state_old -# Rails.logger.info 'TS: ' + state_new.inspect state_old.update_attributes(state_new) else state = Ticket::State.new(state_new) @@ -598,12 +815,10 @@ module Import::OTRS end } end - def self.priority - response = request( 'public.pl?Action=Export;Type=Priority' ) - return if !response - return if !response.success? - result = json(response) + # sync ticket priorities + def self.priority(records) + map = { ChangeTime: :updated_at, CreateTime: :created_at, @@ -615,7 +830,7 @@ module Import::OTRS Comment: :note, } - result.each { |priority| + records.each { |priority| _set_valid(priority) # get new attributes @@ -624,9 +839,8 @@ module Import::OTRS updated_by_id: 1, } map.each { |key, value| - if priority[key.to_s] - priority_new[value] = priority[key.to_s] - end + next if !priority.key?(key.to_s) + priority_new[value] = priority[key.to_s] } # check if state already exists @@ -642,12 +856,9 @@ module Import::OTRS end } end - def self.ticket_group - response = request( 'public.pl?Action=Export;Type=Queue' ) - return if !response - return if !response.success? - result = json(response) + # sync ticket groups / queues + def self.ticket_group(records) map = { ChangeTime: :updated_at, CreateTime: :created_at, @@ -659,7 +870,7 @@ module Import::OTRS Comment: :note, } - result.each { |group| + records.each { |group| _set_valid(group) # get new attributes @@ -668,9 +879,8 @@ module Import::OTRS updated_by_id: 1, } map.each { |key, value| - if group[key.to_s] - group_new[value] = group[key.to_s] - end + next if !group.key?(key.to_s) + group_new[value] = group[key.to_s] } # check if state already exists @@ -686,11 +896,10 @@ module Import::OTRS end } end - def self.user - response = request( 'public.pl?Action=Export;Type=User' ) - return if !response - return if !response.success? - result = json(response) + + # sync agents + def self.user(records, groups, roles, queues) + map = { ChangeTime: :updated_at, CreateTime: :created_at, @@ -706,125 +915,387 @@ module Import::OTRS UserPw: :password, } - result.each { |user| -# Rails.logger.info 'USER: ' + user.inspect + records.each { |user| _set_valid(user) - role = Role.lookup( name: 'Agent' ) + # get roles + role_ids = get_roles_ids(user, groups, roles, queues) + + # get groups + group_ids = get_queue_ids(user, groups, roles, queues) + # get new attributes user_new = { created_by_id: 1, updated_by_id: 1, source: 'OTRS Import', - role_ids: [ role.id ], + role_ids: role_ids, + group_ids: group_ids, } map.each { |key, value| - if user[key.to_s] - user_new[value] = user[key.to_s] - end + next if !user.key?(key.to_s) + user_new[value] = user[key.to_s] } - # check if state already exists + # set pw + if user_new[:password] + user_new[:password] = "{sha2}#{user_new[:password]}" + end + + # check if agent already exists user_old = User.where( id: user_new[:id] ).first - # set state types + # check if login is already used + login_in_use = User.where( "login = ? AND id != #{user_new[:id]}", user_new[:login].downcase ).count + if login_in_use > 0 + user_new[:login] = "#{user_new[:login]}_#{user_new[:id]}" + end + + # create / update agent if user_old - Rails.logger.info "update User.find(#{user_new[:id]})" -# Rails.logger.info 'Update User' + user_new.inspect - user_new.delete( :role_ids ) + log "update User.find(#{user_old[:id]})" + + # only update roles if different (reduce sql statements) + if user_old.role_ids == user_new[:role_ids] + user_new.delete( :role_ids ) + end + user_old.update_attributes(user_new) else - Rails.logger.info "add User.find(#{user_new[:id]})" -# Rails.logger.info 'Add User' + user_new.inspect + log "add User.find(#{user_new[:id]})" user = User.new(user_new) user.id = user_new[:id] user.save end } end - def self.customer - done = false - count = 0 - while done == false - sleep 2 - Rails.logger.info "Count=#{count};Offset=#{count}" - response = request( "public.pl?Action=Export;Type=Customer;Count=100;Offset=#{count}" ) - return if !response - count = count + 3000 - return if !response.success? - result = json(response) - map = { - ChangeTime: :updated_at, - CreateTime: :created_at, - CreateBy: :created_by_id, - ChangeBy: :updated_by_id, - ValidID: :active, - UserComment: :note, - UserEmail: :email, - UserFirstname: :firstname, - UserLastname: :lastname, - UserLogin: :login, - UserPassword: :password, - UserPhone: :phone, - UserFax: :fax, - UserMobile: :mobile, - UserStreet: :street, - UserZip: :zip, - UserCity: :city, - UserCountry: :country, + + def self.get_queue_ids(user, _groups, _roles, queues) + queue_ids = [] + + # lookup by groups + user['GroupIDs'].each {|group_id, permissions| + queues.each {|queue_lookup| + + next if queue_lookup['GroupID'] != group_id + next if !permissions + next if !permissions.include?('rw') + + queue_ids.push queue_lookup['QueueID'] } + } - done = true - result.each { |user| - done = false - _set_valid(user) + # lookup by roles - role = Role.lookup( name: 'Customer' ) + # roles of user + # groups of roles + # queues of group - # get new attributes - user_new = { - created_by_id: 1, - updated_by_id: 1, - source: 'OTRS Import', - role_ids: [role.id], - } - map.each { |key, value| - if user[key.to_s] - user_new[value] = user[key.to_s] - end - } - - # check if state already exists -# user_old = User.where( :login => user_new[:login] ).first - user_old = User.where( login: user_new[:login] ).first - - # set state types - if user_old - Rails.logger.info "update User.find(#{user_new[:id]})" -# Rails.logger.info 'Update User' + user_new.inspect - user_old.update_attributes(user_new) - else -# Rails.logger.info 'Add User' + user_new.inspect - Rails.logger.info "add User.find(#{user_new[:id]})" - user = User.new(user_new) - user.save - end - } - end + queue_ids end + + def self.get_roles_ids(user, groups, roles, _queues) + roles = ['Agent'] + role_ids = [] + user['GroupIDs'].each {|group_id, permissions| + groups.each {|group_lookup| + + next if group_id != group_lookup['ID'] + next if !permissions + + if group_lookup['Name'] == 'admin' && permissions.include?('rw') + roles.push 'Admin' + end + + next if group_lookup['Name'] !~ /^(stats|report)/ + next if !( permissions.include?('ro') || permissions.include?('rw') ) + + roles.push 'Report' + } + } + roles.each {|role| + role_lookup = Role.lookup( name: role ) + next if !role_lookup + role_ids.push role_lookup.id + } + role_ids + end + + # sync customers + + def self.customer(records, organizations) + map = { + ChangeTime: :updated_at, + CreateTime: :created_at, + CreateBy: :created_by_id, + ChangeBy: :updated_by_id, + ValidID: :active, + UserComment: :note, + UserEmail: :email, + UserFirstname: :firstname, + UserLastname: :lastname, + UserLogin: :login, + UserPassword: :password, + UserPhone: :phone, + UserFax: :fax, + UserMobile: :mobile, + UserStreet: :street, + UserZip: :zip, + UserCity: :city, + UserCountry: :country, + } + + role_agent = Role.lookup( name: 'Agent' ) + role_customer = Role.lookup( name: 'Customer' ) + + records.each { |user| + _set_valid(user) + + # get new attributes + user_new = { + created_by_id: 1, + updated_by_id: 1, + source: 'OTRS Import', + organization_id: get_organization_id(user, organizations), + role_ids: [ role_customer.id ], + } + map.each { |key, value| + next if !user.key?(key.to_s) + user_new[value] = user[key.to_s] + } + + # check if customer already exists + user_old = User.where( login: user_new[:login] ).first + + # create / update agent + if user_old + + # do not update user if it is already agent + if !user_old.role_ids.include?( role_agent.id ) + + # only update roles if different (reduce sql statements) + if user_old.role_ids == user_new[:role_ids] + user_new.delete( :role_ids ) + end + log "update User.find(#{user_old[:id]})" + user_old.update_attributes(user_new) + end + else + log "add User.find(#{user_new[:id]})" + user = User.new(user_new) + user.save + end + } + end + + def self.get_organization_id(user, organizations) + organization_id = nil + if user['UserCustomerID'] + organizations.each {|organization| + next if user['UserCustomerID'] != organization['CustomerID'] + organization = Organization.where(name: organization['CustomerCompanyName'] ).first + organization_id = organization.id + } + end + organization_id + end + + # sync organizations + def self.organization(records) + map = { + ChangeTime: :updated_at, + CreateTime: :created_at, + CreateBy: :created_by_id, + ChangeBy: :updated_by_id, + CustomerCompanyName: :name, + ValidID: :active, + CustomerCompanyComment: :note, + } + + records.each { |organization| + _set_valid(organization) + + # get new attributes + organization_new = { + created_by_id: 1, + updated_by_id: 1, + } + map.each { |key, value| + next if !organization.key?(key.to_s) + organization_new[value] = organization[key.to_s] + } + + # check if state already exists + organization_old = Organization.where( name: organization_new[:name] ).first + + # set state types + if organization_old + organization_old.update_attributes(organization_new) + else + organization = Organization.new(organization_new) + organization.id = organization_new[:id] + organization.save + end + } + end + + # sync settings + def self.setting(records) + + records.each { |setting| + + # fqdn + if setting['Key'] == 'FQDN' + Setting.set( 'fqdn', setting['Value'] ) + end + + # http type + if setting['Key'] == 'HttpType' + Setting.set( 'http_type', setting['Value'] ) + end + + # system id + if setting['Key'] == 'SystemID' + Setting.set( 'system_id', setting['Value'] ) + end + + # organization + if setting['Key'] == 'Organization' + Setting.set( 'organization', setting['Value'] ) + end + + # sending emails + if setting['Key'] == 'SendmailModule' + # TODO + end + + # number generater + if setting['Key'] == 'Ticket::NumberGenerator' + if setting['Value'] == 'Kernel::System::Ticket::Number::DateChecksum' + Setting.set( 'ticket_number', 'Ticket::Number::Date' ) + Setting.set( 'ticket_number_date', { checksum: true } ) + elsif setting['Value'] == 'Kernel::System::Ticket::Number::Date' + Setting.set( 'ticket_number', 'Ticket::Number::Date' ) + Setting.set( 'ticket_number_date', { checksum: false } ) + end + end + + # ticket hook + if setting['Key'] == 'Ticket::Hook' + Setting.set( 'ticket_hook', setting['Value'] ) + end + } + end + + # log + def self.log(message) + thread_no = Thread.current[:thread_no] || '-' + Rails.logger.info "thread##{thread_no}: #{message}" + end + + # set translate valid ids to active = true|false def self._set_valid(record) - # map - if record['ValidID'] == '3' - record['ValidID'] = '2' - end - if record['ValidID'] == '2' + + # map + if record['ValidID'].to_s == '3' record['ValidID'] = false - end - if record['ValidID'] == '1' + elsif record['ValidID'].to_s == '2' + record['ValidID'] = false + elsif record['ValidID'].to_s == '1' + record['ValidID'] = true + elsif record['ValidID'].to_s == '0' + record['ValidID'] = false + + # fallback + else record['ValidID'] = true end - if record['ValidID'] == '0' - record['ValidID'] = false + end + + # cleanup invalid values + def self._cleanup(record) + record.each {|key, value| + if value == '0000-00-00 00:00:00' + record[key] = nil + end + } + + # fix OTRS 3.1 bug, no close time if ticket is created + if record['StateType'] == 'closed' && ( !record['Closed'] || record['Closed'].empty? ) + record['Closed'] = record['Created'] end end + + # utf8 convert + def self._utf8_encode(data) + data.each { |key, value| + next if !value + next if value.class != String + data[key] = Encode.conv( 'utf8', value ) + } + end + + # create customers for article + def self._article_based_customers(article, locks) + + # create customer/sender if needed + return if article['sender'] != 'customer' + return if article['created_by_id'].to_i != 1 + return if article['from'].empty? + + email = nil + begin + email = Mail::Address.new( article['from'] ).address + rescue + email = article['from'] + if article['from'] =~ /<(.+?)>/ + email = $1 + end + end + + # create article user if not exists + while locks[:User][ email ] + log "user #{email} is locked" + sleep 1 + end + + # lock user + locks[:User][ email ] = true + + user = User.where( email: email ).first + if !user + user = User.where( login: email ).first + end + if !user + begin + display_name = Mail::Address.new( article['from'] ).display_name || + ( Mail::Address.new( article['from'] ).comments && Mail::Address.new( article['from'] ).comments[0] ) + rescue + display_name = article['from'] + end + + # do extra decoding because we needed to use field.value + display_name = Mail::Field.new( 'X-From', display_name ).to_s + + roles = Role.lookup( name: 'Customer' ) + user = User.create( + login: email, + firstname: display_name, + lastname: '', + email: email, + password: '', + active: true, + role_ids: [roles.id], + updated_by_id: 1, + created_by_id: 1, + ) + end + article['created_by_id'] = user.id + + # unlock user + locks[:User][ email ] = false + + true + end + end diff --git a/lib/import/otrs2.rb b/lib/import/otrs2.rb deleted file mode 100644 index c993572a6..000000000 --- a/lib/import/otrs2.rb +++ /dev/null @@ -1,1301 +0,0 @@ -module Import -end -module Import::OTRS2 - -=begin - - result = request_json( :Subaction => 'List', 1) - - return - - { some json structure } - - result = request_json( :Subaction => 'List' ) - - return - - "some data string" - -=end - - def self.request_json(data, data_only = false) - response = post(data) - if !response - fail "Can't connect to Zammad Migrator" - end - if !response.success? - fail "Can't connect to Zammad Migrator" - end - result = json(response) - if !result - fail 'Invalid response' - end - if data_only - result['Result'] - else - result - end - end - -=begin - - start get request to backend, add auth data automatically - - result = request('Subaction=List') - - return - - "some data string" - -=end - - def self.request(part) - url = Setting.get('import_otrs_endpoint') + part + ';Key=' + Setting.get('import_otrs_endpoint_key') - log 'GET: ' + url - response = UserAgent.get( - url, - {}, - { - open_timeout: 10, - read_timeout: 60, - user: Setting.get('import_otrs_user'), - password: Setting.get('import_otrs_password'), - }, - ) - if !response.success? - log "ERROR: #{response.error}" - return - end - response - end - -=begin - - start post request to backend, add auth data automatically - - result = request('Subaction=List') - - return - - "some data string" - -=end - - def self.post(data, url = nil) - if !url - url = Setting.get('import_otrs_endpoint') - data['Action'] = 'ZammadMigrator' - end - data['Key'] = Setting.get('import_otrs_endpoint_key') - log 'POST: ' + url - log 'PARAMS: ' + data.inspect - open_timeout = 10 - read_timeout = 120 - if data.empty? - open_timeout = 6 - read_timeout = 20 - end - response = UserAgent.post( - url, - data, - { - open_timeout: open_timeout, - read_timeout: read_timeout, - user: Setting.get('import_otrs_user'), - password: Setting.get('import_otrs_password'), - }, - ) - if !response.success? - log "ERROR: #{response.error}" - return - end - response - end - -=begin - - start post request to backend, add auth data automatically - - result = json('some response string') - - return - - {} - -=end - - def self.json(response) - data = Encode.conv( 'utf8', response.body.to_s ) - JSON.parse( data ) - end - -=begin - - start auth on OTRS - just for experimental reasons - - result = auth(username, password) - - return - - { ..user structure.. } - -=end - - def self.auth(username, password) - url = Setting.get('import_otrs_endpoint') - url.gsub!('ZammadMigrator', 'ZammadSSO') - response = post( { Action: 'ZammadSSO', Subaction: 'Auth', User: username, Pw: password }, url ) - return if !response - return if !response.success? - - result = json(response) - result - end - -=begin - - request session data - just for experimental reasons - - result = session(session_id) - - return - - { ..session structure.. } - -=end - - def self.session(session_id) - url = Setting.get('import_otrs_endpoint') - url.gsub!('ZammadMigrator', 'ZammadSSO') - response = post( { Action: 'ZammadSSO', Subaction: 'SessionCheck', SessionID: session_id }, url ) - return if !response - return if !response.success? - result = json(response) - result - end - -=begin - - load objects from otrs - - result = load('SysConfig') - - return - - [ - { ..object1.. }, - { ..object2.. }, - { ..object3.. }, - ] - -=end - - def self.load( object, limit = '', offset = '', diff = 0 ) - request_json( { Subaction: 'Export', Object: object, Limit: limit, Offset: offset, Diff: diff }, 1 ) - end - -=begin - - start get request to backend to check connection - - result = connection_test - - return - - true | false - -=end - - def self.connection_test - self.request_json({}) - end - -=begin - - get object statistic from server ans save it in cache - - result = statistic('Subaction=List') - - return - - { - 'Ticket' => 1234, - 'User' => 123, - 'SomeObject' => 999, - } - -=end - - def self.statistic - - # check cache - cache = Cache.get('import_otrs_stats') - if cache - return cache - end - - # retrive statistic - statistic = self.request_json( { Subaction: 'List' }, 1) - if statistic - Cache.write('import_otrs_stats', statistic) - end - statistic - end - -=begin - - return current import state - - result = current_state - - return - - { - :Ticket => { - :total => 1234, - :done => 13, - }, - :Base => { - :total => 1234, - :done => 13, - }, - } - -=end - - def self.current_state - data = self.statistic - base = Group.count + Ticket::State.count + Ticket::Priority.count - base_total = data['Queue'] + data['State'] + data['Priority'] - user = User.count - user_total = data['User'] + data['CustomerUser'] - data = { - Base: { - done: base, - total: base_total || 0, - }, - User: { - done: user, - total: user_total || 0, - }, - Ticket: { - done: Ticket.count, - total: data['Ticket'] || 0, - }, - } - data - end - - # - # start import - # - # Import::OTRS2.start - # - - def self.start - log 'Start import...' - - # check if system is in import mode - if !Setting.get('import_mode') - fail 'System is not in import mode!' - end - - result = request_json({}) - if !result['Success'] - 'API key not valid!' - end - - # set settings - settings = load('SysConfig') - setting(settings) - - # dynamic fields - dynamic_fields = load('DynamicField') - #settings(dynamic_fields, settings) - - # email accounts - #accounts = load('PostMasterAccount') - #account(accounts) - - # email filter - #filters = load('PostMasterFilter') - #filter(filters) - - # create states - states = load('State') - ActiveRecord::Base.transaction do - state(states) - end - - # create priorities - priorities = load('Priority') - ActiveRecord::Base.transaction do - priority(priorities) - end - - # create groups - queues = load('Queue') - ActiveRecord::Base.transaction do - ticket_group(queues) - end - - # get agents groups - groups = load('Group') - - # get agents roles - roles = load('Role') - - # create agents - users = load('User') - ActiveRecord::Base.transaction do - user(users, groups, roles, queues) - end - - # create organizations - organizations = load('Customer') - ActiveRecord::Base.transaction do - organization(organizations) - end - - # create customers - count = 0 - steps = 50 - run = true - while run - count += steps - records = load('CustomerUser', steps, count - steps) - if !records || !records[0] - log 'all customers imported.' - run = false - next - end - customer(records, organizations) - end - - Thread.abort_on_exception = true - thread_count = 8 - threads = {} - count = 0 - locks = { User: {} } - (1..thread_count).each {|thread| - threads[thread] = Thread.new { - Thread.current[:thread_no] = thread - sleep thread * 3 - log "Started import thread# #{thread} ..." - run = true - steps = 20 - while run - count += steps - log "loading... thread# #{thread} ..." - offset = count - steps - if offset != 0 - offset = count - steps + 1 - end - records = load( 'Ticket', steps, count - steps) - if !records || !records[0] - log "... thread# #{thread}, no more work." - run = false - next - end - _ticket_result(records, locks, thread) - end - ActiveRecord::Base.connection.close - } - } - (1..thread_count).each {|thread| - threads[thread].join - } - - Setting.set( 'system_init_done', true ) - #Setting.set( 'import_mode', false ) - - true - end - - def self.diff_worker - return if !Setting.get('import_mode') - return if Setting.get('import_otrs_endpoint') == 'http://otrs_host/otrs' - self.diff - end - - def self.diff - log 'Start diff...' - - # check if system is in import mode - if !Setting.get('import_mode') - fail 'System is not in import mode!' - end - - # create states - states = load('State') - state(states) - - # create priorities - priorities = load('Priority') - priority(priorities) - - # create groups - queues = load('Queue') - ticket_group(queues) - - # get agents groups - groups = load('Group') - - # get agents roles - roles = load('Role') - - # create agents - users = load('User') - user(users, groups, roles, queues) - - # create organizations - organizations = load('Customer') - organization(organizations) - - # get changed tickets - self.ticket_diff - - end - - def self.ticket_diff - count = 0 - run = true - steps = 20 - locks = { User: {} } - while run - count += steps - log 'loading... diff ...' - offset = count - steps - if offset != 0 - offset = count - steps + 1 - end - records = load( 'Ticket', steps, count - steps, 1 ) - if !records || !records[0] - log '... no more work.' - run = false - next - end - _ticket_result(records, locks) - end - - end - - def self._ticket_result(result, locks, _thread = '-') -# puts result.inspect - map = { - Ticket: { - Changed: :updated_at, - Created: :created_at, - CreateBy: :created_by_id, - TicketNumber: :number, - QueueID: :group_id, - StateID: :state_id, - PriorityID: :priority_id, - Owner: :owner, - CustomerUserID: :customer, - Title: :title, - TicketID: :id, - FirstResponse: :first_response, -# :FirstResponseTimeDestinationDate => :first_response_escal_date, -# :FirstResponseInMin => :first_response_in_min, -# :FirstResponseDiffInMin => :first_response_diff_in_min, - Closed: :close_time, -# :SoltutionTimeDestinationDate => :close_time_escal_date, -# :CloseTimeInMin => :close_time_in_min, -# :CloseTimeDiffInMin => :close_time_diff_in_min, - }, - Article: { - SenderType: :sender, - ArticleType: :type, - TicketID: :ticket_id, - ArticleID: :id, - Body: :body, - From: :from, - To: :to, - Cc: :cc, - Subject: :subject, - InReplyTo: :in_reply_to, - MessageID: :message_id, -# :ReplyTo => :reply_to, - References: :references, - Changed: :updated_at, - Created: :created_at, - ChangedBy: :updated_by_id, - CreatedBy: :created_by_id, - }, - } - - result.each {|record| - - # cleanup values - _cleanup(record) - - _utf8_encode(record) - - ticket_new = { - title: '', - created_by_id: 1, - updated_by_id: 1, - } - map[:Ticket].each { |key, value| - next if !record.key?(key.to_s) - ticket_new[value] = record[key.to_s] - } - ticket_old = Ticket.where( id: ticket_new[:id] ).first - - # find owner - if ticket_new[:owner] - user = User.lookup( login: ticket_new[:owner].downcase ) - if user - ticket_new[:owner_id] = user.id - else - ticket_new[:owner_id] = 1 - end - ticket_new.delete(:owner) - end - - # find customer - if ticket_new[:customer] - user = User.lookup( login: ticket_new[:customer].downcase ) - if user - ticket_new[:customer_id] = user.id - else - ticket_new[:customer_id] = 1 - end - ticket_new.delete(:customer) - else - ticket_new[:customer_id] = 1 - end - - # set state types - if ticket_old - log "update Ticket.find(#{ticket_new[:id]})" - ticket_old.update_attributes(ticket_new) - else - log "add Ticket.find(#{ticket_new[:id]})" - ticket = Ticket.new(ticket_new) - ticket.id = ticket_new[:id] - ticket.save - end - - # utf8 encode - record['Articles'].each { |article| - _utf8_encode(article) - } - - # lookup customers to create first - record['Articles'].each { |article| - _article_based_customers(article, locks) - } - - ActiveRecord::Base.transaction do - record['Articles'].each { |article| - - # get article values - article_new = { - created_by_id: 1, - updated_by_id: 1, - } - - map[:Article].each { |key, value| - next if !article.key?(key.to_s) - article_new[value] = article[key.to_s] - } - - if article_new[:sender] == 'customer' - article_new[:sender_id] = Ticket::Article::Sender.lookup( name: 'Customer' ).id - article_new.delete( :sender ) - end - if article_new[:sender] == 'agent' - article_new[:sender_id] = Ticket::Article::Sender.lookup( name: 'Agent' ).id - article_new.delete( :sender ) - end - if article_new[:sender] == 'system' - article_new[:sender_id] = Ticket::Article::Sender.lookup( name: 'System' ).id - article_new.delete( :sender ) - end - - if article_new[:type] == 'email-external' - article_new[:type_id] = Ticket::Article::Type.lookup( name: 'email' ).id - article_new[:internal] = false - elsif article_new[:type] == 'email-internal' - article_new[:type_id] = Ticket::Article::Type.lookup( name: 'email' ).id - article_new[:internal] = true - elsif article_new[:type] == 'note-external' - article_new[:type_id] = Ticket::Article::Type.lookup( name: 'note' ).id - article_new[:internal] = false - elsif article_new[:type] == 'note-internal' - article_new[:type_id] = Ticket::Article::Type.lookup( name: 'note' ).id - article_new[:internal] = true - elsif article_new[:type] == 'phone' - article_new[:type_id] = Ticket::Article::Type.lookup( name: 'phone' ).id - article_new[:internal] = false - elsif article_new[:type] == 'webrequest' - article_new[:type_id] = Ticket::Article::Type.lookup( name: 'web' ).id - article_new[:internal] = false - else - article_new[:type_id] = 9 - end - article_new.delete( :type ) - article_old = Ticket::Article.where( id: article_new[:id] ).first - - # set state types - if article_old - log "update Ticket::Article.find(#{article_new[:id]})" - article_old.update_attributes(article_new) - else - log "add Ticket::Article.find(#{article_new[:id]})" - article = Ticket::Article.new(article_new) - article.id = article_new[:id] - article.save - end - } - end - - #puts "HS: #{record['History'].inspect}" - record['History'].each { |history| - if history['HistoryType'] == 'NewTicket' - #puts "HS.add( #{history.inspect} )" - res = History.add( - id: history['HistoryID'], - o_id: history['TicketID'], - history_type: 'created', - history_object: 'Ticket', - created_at: history['CreateTime'], - created_by_id: history['CreateBy'] - ) - #puts "res #{res.inspect}" - end - if history['HistoryType'] == 'StateUpdate' - data = history['Name'] - # "%%new%%open%%" - from = nil - to = nil - if data =~ /%%(.+?)%%(.+?)%%/ - from = $1 - to = $2 - state_from = Ticket::State.lookup( name: from ) - state_to = Ticket::State.lookup( name: to ) - if state_from - from_id = state_from.id - end - if state_to - to_id = state_to.id - end - end - History.add( - id: history['HistoryID'], - o_id: history['TicketID'], - history_type: 'updated', - history_object: 'Ticket', - history_attribute: 'state', - value_from: from, - id_from: from_id, - value_to: to, - id_to: to_id, - created_at: history['CreateTime'], - created_by_id: history['CreateBy'] - ) - end - if history['HistoryType'] == 'Move' - data = history['Name'] - # "%%Queue1%%5%%Postmaster%%1" - from = nil - to = nil - if data =~ /%%(.+?)%%(.+?)%%(.+?)%%(.+?)$/ - from = $1 - from_id = $2 - to = $3 - to_id = $4 - end - History.add( - id: history['HistoryID'], - o_id: history['TicketID'], - history_type: 'updated', - history_object: 'Ticket', - history_attribute: 'group', - value_from: from, - value_to: to, - id_from: from_id, - id_to: to_id, - created_at: history['CreateTime'], - created_by_id: history['CreateBy'] - ) - end - if history['HistoryType'] == 'PriorityUpdate' - data = history['Name'] - # "%%3 normal%%3%%5 very high%%5" - from = nil - to = nil - if data =~ /%%(.+?)%%(.+?)%%(.+?)%%(.+?)$/ - from = $1 - from_id = $2 - to = $3 - to_id = $4 - end - History.add( - id: history['HistoryID'], - o_id: history['TicketID'], - history_type: 'updated', - history_object: 'Ticket', - history_attribute: 'priority', - value_from: from, - value_to: to, - id_from: from_id, - id_to: to_id, - created_at: history['CreateTime'], - created_by_id: history['CreateBy'] - ) - end - - next if !history['ArticleID'] - next if history['ArticleID'] == 0 - - History.add( - id: history['HistoryID'], - o_id: history['ArticleID'], - history_type: 'created', - history_object: 'Ticket::Article', - related_o_id: history['TicketID'], - related_history_object: 'Ticket', - created_at: history['CreateTime'], - created_by_id: history['CreateBy'] - ) - } - } - end - - # sync ticket states - def self.state(records) - map = { - ChangeTime: :updated_at, - CreateTime: :created_at, - CreateBy: :created_by_id, - ChangeBy: :updated_by_id, - Name: :name, - ID: :id, - ValidID: :active, - Comment: :note, - } - - # rename states to handle not uniq issues - Ticket::State.all.each {|state| - state.name = state.name + '_tmp' - state.save - } - - records.each { |state| - _set_valid(state) - - # get new attributes - state_new = { - created_by_id: 1, - updated_by_id: 1, - } - map.each { |key, value| - next if !state.key?(key.to_s) - state_new[value] = state[key.to_s] - } - - # check if state already exists - state_old = Ticket::State.where( id: state_new[:id] ).first - - # set state types - if state['TypeName'] == 'pending auto' - state['TypeName'] = 'pending action' - end - state_type = Ticket::StateType.where( name: state['TypeName'] ).first - state_new[:state_type_id] = state_type.id - if state_old - state_old.update_attributes(state_new) - else - state = Ticket::State.new(state_new) - state.id = state_new[:id] - state.save - end - } - end - - # sync ticket priorities - def self.priority(records) - - map = { - ChangeTime: :updated_at, - CreateTime: :created_at, - CreateBy: :created_by_id, - ChangeBy: :updated_by_id, - Name: :name, - ID: :id, - ValidID: :active, - Comment: :note, - } - - records.each { |priority| - _set_valid(priority) - - # get new attributes - priority_new = { - created_by_id: 1, - updated_by_id: 1, - } - map.each { |key, value| - next if !priority.key?(key.to_s) - priority_new[value] = priority[key.to_s] - } - - # check if state already exists - priority_old = Ticket::Priority.where( id: priority_new[:id] ).first - - # set state types - if priority_old - priority_old.update_attributes(priority_new) - else - priority = Ticket::Priority.new(priority_new) - priority.id = priority_new[:id] - priority.save - end - } - end - - # sync ticket groups / queues - def self.ticket_group(records) - map = { - ChangeTime: :updated_at, - CreateTime: :created_at, - CreateBy: :created_by_id, - ChangeBy: :updated_by_id, - Name: :name, - QueueID: :id, - ValidID: :active, - Comment: :note, - } - - records.each { |group| - _set_valid(group) - - # get new attributes - group_new = { - created_by_id: 1, - updated_by_id: 1, - } - map.each { |key, value| - next if !group.key?(key.to_s) - group_new[value] = group[key.to_s] - } - - # check if state already exists - group_old = Group.where( id: group_new[:id] ).first - - # set state types - if group_old - group_old.update_attributes(group_new) - else - group = Group.new(group_new) - group.id = group_new[:id] - group.save - end - } - end - - # sync agents - def self.user(records, groups, roles, queues) - - map = { - ChangeTime: :updated_at, - CreateTime: :created_at, - CreateBy: :created_by_id, - ChangeBy: :updated_by_id, - UserID: :id, - ValidID: :active, - Comment: :note, - UserEmail: :email, - UserFirstname: :firstname, - UserLastname: :lastname, - UserLogin: :login, - UserPw: :password, - } - - records.each { |user| - _set_valid(user) - - # get roles - role_ids = get_roles_ids(user, groups, roles, queues) - - # get groups - group_ids = get_queue_ids(user, groups, roles, queues) - - # get new attributes - user_new = { - created_by_id: 1, - updated_by_id: 1, - source: 'OTRS Import', - role_ids: role_ids, - group_ids: group_ids, - } - map.each { |key, value| - next if !user.key?(key.to_s) - user_new[value] = user[key.to_s] - } - - # set pw - if user_new[:password] - user_new[:password] = "{sha2}#{user_new[:password]}" - end - - # check if agent already exists - user_old = User.where( id: user_new[:id] ).first - - # check if login is already used - login_in_use = User.where( "login = ? AND id != #{user_new[:id]}", user_new[:login].downcase ).count - if login_in_use > 0 - user_new[:login] = "#{user_new[:login]}_#{user_new[:id]}" - end - - # create / update agent - if user_old - log "update User.find(#{user_old[:id]})" - - # only update roles if different (reduce sql statements) - if user_old.role_ids == user_new[:role_ids] - user_new.delete( :role_ids ) - end - - user_old.update_attributes(user_new) - else - log "add User.find(#{user_new[:id]})" - user = User.new(user_new) - user.id = user_new[:id] - user.save - end - } - end - - def self.get_queue_ids(user, _groups, _roles, queues) - queue_ids = [] - - # lookup by groups - user['GroupIDs'].each {|group_id, permissions| - queues.each {|queue_lookup| - - next if queue_lookup['GroupID'] != group_id - next if !permissions - next if !permissions.include?('rw') - - queue_ids.push queue_lookup['QueueID'] - } - } - - # lookup by roles - - # roles of user - # groups of roles - # queues of group - - queue_ids - end - - def self.get_roles_ids(user, groups, roles, _queues) - roles = ['Agent'] - role_ids = [] - user['GroupIDs'].each {|group_id, permissions| - groups.each {|group_lookup| - - next if group_id != group_lookup['ID'] - next if !permissions - - if group_lookup['Name'] == 'admin' && permissions.include?('rw') - roles.push 'Admin' - end - - next if group_lookup['Name'] !~ /^(stats|report)/ - next if !( permissions.include?('ro') || permissions.include?('rw') ) - - roles.push 'Report' - } - } - roles.each {|role| - role_lookup = Role.lookup( name: role ) - next if !role_lookup - role_ids.push role_lookup.id - } - role_ids - end - - # sync customers - - def self.customer(records, organizations) - map = { - ChangeTime: :updated_at, - CreateTime: :created_at, - CreateBy: :created_by_id, - ChangeBy: :updated_by_id, - ValidID: :active, - UserComment: :note, - UserEmail: :email, - UserFirstname: :firstname, - UserLastname: :lastname, - UserLogin: :login, - UserPassword: :password, - UserPhone: :phone, - UserFax: :fax, - UserMobile: :mobile, - UserStreet: :street, - UserZip: :zip, - UserCity: :city, - UserCountry: :country, - } - - role_agent = Role.lookup( name: 'Agent' ) - role_customer = Role.lookup( name: 'Customer' ) - - records.each { |user| - _set_valid(user) - - # get new attributes - user_new = { - created_by_id: 1, - updated_by_id: 1, - source: 'OTRS Import', - organization_id: get_organization_id(user, organizations), - role_ids: [ role_customer.id ], - } - map.each { |key, value| - next if !user.key?(key.to_s) - user_new[value] = user[key.to_s] - } - - # check if customer already exists - user_old = User.where( login: user_new[:login] ).first - - # create / update agent - if user_old - - # do not update user if it is already agent - if !user_old.role_ids.include?( role_agent.id ) - - # only update roles if different (reduce sql statements) - if user_old.role_ids == user_new[:role_ids] - user_new.delete( :role_ids ) - end - log "update User.find(#{user_old[:id]})" - user_old.update_attributes(user_new) - end - else - log "add User.find(#{user_new[:id]})" - user = User.new(user_new) - user.save - end - } - end - - def self.get_organization_id(user, organizations) - organization_id = nil - if user['UserCustomerID'] - organizations.each {|organization| - next if user['UserCustomerID'] != organization['CustomerID'] - organization = Organization.where(name: organization['CustomerCompanyName'] ).first - organization_id = organization.id - } - end - organization_id - end - - # sync organizations - def self.organization(records) - map = { - ChangeTime: :updated_at, - CreateTime: :created_at, - CreateBy: :created_by_id, - ChangeBy: :updated_by_id, - CustomerCompanyName: :name, - ValidID: :active, - CustomerCompanyComment: :note, - } - - records.each { |organization| - _set_valid(organization) - - # get new attributes - organization_new = { - created_by_id: 1, - updated_by_id: 1, - } - map.each { |key, value| - next if !organization.key?(key.to_s) - organization_new[value] = organization[key.to_s] - } - - # check if state already exists - organization_old = Organization.where( name: organization_new[:name] ).first - - # set state types - if organization_old - organization_old.update_attributes(organization_new) - else - organization = Organization.new(organization_new) - organization.id = organization_new[:id] - organization.save - end - } - end - - # sync settings - def self.setting(records) - - records.each { |setting| - - # fqdn - if setting['Key'] == 'FQDN' - Setting.set( 'fqdn', setting['Value'] ) - end - - # http type - if setting['Key'] == 'HttpType' - Setting.set( 'http_type', setting['Value'] ) - end - - # system id - if setting['Key'] == 'SystemID' - Setting.set( 'system_id', setting['Value'] ) - end - - # organization - if setting['Key'] == 'Organization' - Setting.set( 'organization', setting['Value'] ) - end - - # sending emails - if setting['Key'] == 'SendmailModule' - # TODO - end - - # number generater - if setting['Key'] == 'Ticket::NumberGenerator' - if setting['Value'] == 'Kernel::System::Ticket::Number::DateChecksum' - Setting.set( 'ticket_number', 'Ticket::Number::Date' ) - Setting.set( 'ticket_number_date', { checksum: true } ) - elsif setting['Value'] == 'Kernel::System::Ticket::Number::Date' - Setting.set( 'ticket_number', 'Ticket::Number::Date' ) - Setting.set( 'ticket_number_date', { checksum: false } ) - end - end - - # ticket hook - if setting['Key'] == 'Ticket::Hook' - Setting.set( 'ticket_hook', setting['Value'] ) - end - } - end - - # log - def self.log(message) - thread_no = Thread.current[:thread_no] || '-' - Rails.logger.info "thread##{thread_no}: #{message}" - end - - # set translate valid ids to active = true|false - def self._set_valid(record) - - # map - if record['ValidID'].to_s == '3' - record['ValidID'] = false - elsif record['ValidID'].to_s == '2' - record['ValidID'] = false - elsif record['ValidID'].to_s == '1' - record['ValidID'] = true - elsif record['ValidID'].to_s == '0' - record['ValidID'] = false - - # fallback - else - record['ValidID'] = true - end - end - - # cleanup invalid values - def self._cleanup(record) - record.each {|key, value| - if value == '0000-00-00 00:00:00' - record[key] = nil - end - } - - # fix OTRS 3.1 bug, no close time if ticket is created - if record['StateType'] == 'closed' && ( !record['Closed'] || record['Closed'].empty? ) - record['Closed'] = record['Created'] - end - end - - # utf8 convert - def self._utf8_encode(data) - data.each { |key, value| - next if !value - next if value.class != String - data[key] = Encode.conv( 'utf8', value ) - } - end - - # create customers for article - def self._article_based_customers(article, locks) - - # create customer/sender if needed - return if article['sender'] != 'customer' - return if article['created_by_id'].to_i != 1 - return if article['from'].empty? - - email = nil - begin - email = Mail::Address.new( article['from'] ).address - rescue - email = article['from'] - if article['from'] =~ /<(.+?)>/ - email = $1 - end - end - - # create article user if not exists - while locks[:User][ email ] - log "user #{email} is locked" - sleep 1 - end - - # lock user - locks[:User][ email ] = true - - user = User.where( email: email ).first - if !user - user = User.where( login: email ).first - end - if !user - begin - display_name = Mail::Address.new( article['from'] ).display_name || - ( Mail::Address.new( article['from'] ).comments && Mail::Address.new( article['from'] ).comments[0] ) - rescue - display_name = article['from'] - end - - # do extra decoding because we needed to use field.value - display_name = Mail::Field.new( 'X-From', display_name ).to_s - - roles = Role.lookup( name: 'Customer' ) - user = User.create( - login: email, - firstname: display_name, - lastname: '', - email: email, - password: '', - active: true, - role_ids: [roles.id], - updated_by_id: 1, - created_by_id: 1, - ) - end - article['created_by_id'] = user.id - - # unlock user - locks[:User][ email ] = false - - true - end - -end diff --git a/lib/sso/otrs.rb b/lib/sso/otrs.rb index be74e3b26..b97a0a5e6 100644 --- a/lib/sso/otrs.rb +++ b/lib/sso/otrs.rb @@ -10,7 +10,7 @@ module Sso::Otrs return false if !params['SessionID'] # connect to OTRS - result = Import::OTRS2.session( params['SessionID'] ) + result = Import::OTRS.session( params['SessionID'] ) return false if !result return false if !result['groups_ro'] diff --git a/test/integration/otrs_import_test.rb b/test/integration/otrs_import_test.rb index c1c509a03..fd51fa511 100644 --- a/test/integration/otrs_import_test.rb +++ b/test/integration/otrs_import_test.rb @@ -13,7 +13,7 @@ class OtrsImportTest < ActiveSupport::TestCase Setting.set('import_otrs_endpoint', ENV['IMPORT_OTRS_ENDPOINT']) Setting.set('import_otrs_endpoint_key', ENV['IMPORT_OTRS_ENDPOINT_KEY']) Setting.set('import_mode', true) - Import::OTRS2.start + Import::OTRS.start # check settings items test 'check settings' do