From 64d722d14cf56ea3ffb6a21c320ebf37d9c6ae9d Mon Sep 17 00:00:00 2001 From: Thorsten Eckel Date: Mon, 18 Jan 2016 20:32:34 +0100 Subject: [PATCH] Added zendesk import browser tests. --- .gitlab-ci.yml | 10 + .../app/controllers/import_zendesk.coffee | 101 ++++++---- .../app/views/import/zendesk.jst.eco | 108 +++++++++++ app/controllers/import_zendesk_controller.rb | 124 +++++++++++++ config/routes/import_zendesk.rb | 10 + lib/import/zendesk.rb | 175 ++++++++++++++++-- .../zendesk_import_browser_test.rb | 87 +++++++++ 7 files changed, 565 insertions(+), 50 deletions(-) create mode 100644 app/assets/javascripts/app/views/import/zendesk.jst.eco create mode 100644 app/controllers/import_zendesk_controller.rb create mode 100644 config/routes/import_zendesk.rb create mode 100644 test/integration/zendesk_import_browser_test.rb diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f3f18b3f0..8afae5d17 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -404,3 +404,13 @@ job_integration_autowizard_chrome: - ruby -I test/ test/integration/auto_wizard_test.rb || script/build/test_shutdown.sh $RAILS_ENV $BROWSER_PORT $WS_PORT 1 - script/build/test_shutdown.sh $RAILS_ENV $BROWSER_PORT $WS_PORT +job_integration_zendesk_chrome: + - export BROWSER_PORT=4071 + - export WS_PORT=4072 + - export BROWSER_URL=http://$IP:$BROWSER_PORT + - RAILS_ENV=test rake db:create + - script/bootstrap.sh + - rake assets:precompile + - script/build/test_startup.sh $RAILS_ENV $BROWSER_PORT $WS_PORT + - ruby -I test/ test/integration/zendesk_import_browser_test.rb || script/build/test_shutdown.sh $RAILS_ENV $BROWSER_PORT $WS_PORT 1 + - script/build/test_shutdown.sh $RAILS_ENV $BROWSER_PORT $WS_PORT diff --git a/app/assets/javascripts/app/controllers/import_zendesk.coffee b/app/assets/javascripts/app/controllers/import_zendesk.coffee index 8be4edd9d..3b9f66b1b 100644 --- a/app/assets/javascripts/app/controllers/import_zendesk.coffee +++ b/app/assets/javascripts/app/controllers/import_zendesk.coffee @@ -1,14 +1,21 @@ class Index extends App.ControllerContent className: 'getstarted fit' elements: - '.input-feedback': 'urlStatus' - '[data-target=otrs-start-migration]': 'nextStartMigration' - '.otrs-link-error': 'linkErrorMessage' + '.input-feedback': 'urlStatus' + '[data-target=zendesk-credentials]': 'nextEnterCredentials' + '[data-target=zendesk-start-migration]': 'nextStartMigration' + '#zendesk-url': 'zendeskUrl' + '.js-zendeskUrlApiToken': 'zendeskUrlApiToken' + '.zendesk-url-error': 'linkErrorMessage' + '.zendesk-api-token-error': 'apiTokenErrorMessage' + '#zendesk-email': 'zendeskEmail' + '#zendesk-api-token': 'zendeskApiToken' + events: - 'click .js-otrs-link': 'showLink' - 'click .js-download': 'startDownload' - 'click .js-migration-start': 'startMigration' - 'keyup #otrs-link': 'updateUrl' + 'click .js-zendesk-credentials': 'showCredentials' + 'click .js-migration-start': 'startMigration' + 'keyup #zendesk-url': 'updateUrl' + 'keyup #zendesk-api-token': 'updateApiToken' constructor: -> super @@ -34,7 +41,7 @@ class Index extends App.ControllerContent return # check if import is active - if data.import_mode == true && data.import_backend != 'otrs' + if data.import_mode == true && data.import_backend != 'zendesk' @navigate '#import/' + data.import_backend return @@ -47,34 +54,19 @@ class Index extends App.ControllerContent ) render: -> - @html App.view('import/otrs')() - - startDownload: (e) => - e.preventDefault() - @$('.js-otrs-link').removeClass('hide') - - showLink: (e) => - e.preventDefault() - @$('[data-slide=otrs-plugin]').toggleClass('hide') - @$('[data-slide=otrs-link]').toggleClass('hide') - - showImportState: => - @$('[data-slide=otrs-plugin]').addClass('hide') - @$('[data-slide=otrs-link]').addClass('hide') - @$('[data-slide=otrs-import]').removeClass('hide') + @html App.view('import/zendesk')() updateUrl: (e) => - url = $(e.target).val() @urlStatus.attr('data-state', 'loading') @linkErrorMessage.text('') # get data callback = => @ajax( - id: 'import_otrs_url', + id: 'import_zendesk_url', type: 'POST', - url: @apiPath + '/import/otrs/url_check', - data: JSON.stringify(url: url) + url: @apiPath + '/import/zendesk/url_check', + data: JSON.stringify(url: @zendeskUrl.val()) processData: true, success: (data, status, xhr) => @@ -83,14 +75,55 @@ class Index extends App.ControllerContent if data.result is 'ok' @urlStatus.attr('data-state', 'success') @linkErrorMessage.text('') - @nextStartMigration.removeClass('hide') + @nextEnterCredentials.removeClass('hide') else @urlStatus.attr('data-state', 'error') @linkErrorMessage.text( data.message_human || data.message ) + @nextEnterCredentials.addClass('hide') + + ) + @delay( callback, 700, 'import_zendesk_url' ) + + updateApiToken: (e) => + @urlStatus.attr('data-state', 'loading') + @apiTokenErrorMessage.text('') + + # get data + callback = => + @ajax( + id: 'import_zendesk_api_token', + type: 'POST', + url: @apiPath + '/import/zendesk/credentials_check', + data: JSON.stringify(username: @zendeskEmail.val(), token: @zendeskApiToken.val()) + processData: true, + success: (data, status, xhr) => + + # validate form + console.log(data) + if data.result is 'ok' + @urlStatus.attr('data-state', 'success') + @apiTokenErrorMessage.text('') + @nextStartMigration.removeClass('hide') + else + @urlStatus.attr('data-state', 'error') + @apiTokenErrorMessage.text( data.message_human || data.message ) @nextStartMigration.addClass('hide') ) - @delay( callback, 700, 'import_otrs_url' ) + @delay( callback, 700, 'import_zendesk_api_token' ) + + showCredentials: (e) => + e.preventDefault() + @urlStatus.attr('data-state', '') + @zendeskUrlApiToken.attr('href', @zendeskUrl.val() + "agent/admin/api") + @zendeskUrlApiToken.val('HERE') + @$('[data-slide=zendesk-url]').toggleClass('hide') + @$('[data-slide=zendesk-credentials]').toggleClass('hide') + + showImportState: => + @$('[data-slide=zendesk-url]').addClass('hide') + @$('[data-slide=zendesk-credentials]').addClass('hide') + @$('[data-slide=zendesk-import]').removeClass('hide') startMigration: (e) => e.preventDefault() @@ -98,7 +131,7 @@ class Index extends App.ControllerContent @ajax( id: 'import_start', type: 'POST', - url: @apiPath + '/import/otrs/import_start', + url: @apiPath + '/import/zendesk/import_start', processData: true, success: (data, status, xhr) => @@ -108,19 +141,17 @@ class Index extends App.ControllerContent @delay( @updateMigration, 3000 ) ) - updateMigration: => @showImportState() @ajax( id: 'import_status', type: 'GET', - url: @apiPath + '/import/otrs/import_status', + url: @apiPath + '/import/zendesk/import_status', processData: true, success: (data, status, xhr) => - if data.setup_done - @Config.set('system_init_done', true) - @navigate '#' + if data.result is 'import_done' + window.location.reload() return for key, item of data.data diff --git a/app/assets/javascripts/app/views/import/zendesk.jst.eco b/app/assets/javascripts/app/views/import/zendesk.jst.eco new file mode 100644 index 000000000..32b875196 --- /dev/null +++ b/app/assets/javascripts/app/views/import/zendesk.jst.eco @@ -0,0 +1,108 @@ +
+ <%- @Icon('full-logo', 'wizard-logo') %> +
+
+

<%- @T('Zendesk URL') %>

+
+

+ <%- @T('Enter the URL of your Zendesk system') %>: +

+
+ +
+ +
+
+ <%- @Icon('diagonal-cross', 'icon-error') %> + <%- @Icon('checkmark') %> +
+
+
+
+
+
+ <%- @T('Go Back') %> +
<%- @T('Enter credentials') %>
+
+
+ +
+

<%- @T('Zendesk credentials') %>

+
+

+ <%- @T('Enter your Email address and the Zendesk API token gained from your') %> <%- @T('admin interface') %> + +

+
+ +
+ +
+ +
+ +
+
+ <%- @Icon('diagonal-cross', 'icon-error') %> + <%- @Icon('checkmark') %> +
+
+
+
+
+
+ <%- @T('Go Back') %> +
<%- @T('Migrate Zendesk Data') %>
+
+
+ +
+

<%- @T('Zendesk Migration') %>

+ +
+ + + + + + + + + +
-/- + <%- @T('Groups') %> + +
+
+ <%- @Icon('checkmark') %> +
+
-/- + <%- @T('Organizations') %> + +
+
+ <%- @Icon('checkmark') %> +
+
-/- + <%- @T('Users') %> + +
+
+ <%- @Icon('checkmark') %> +
+
-/- + <%- @T('Tickets') %> + +
+
+ <%- @Icon('checkmark') %> +
+
+
+
+ <%- @T('Done') %> +
+
+ +
+
\ No newline at end of file diff --git a/app/controllers/import_zendesk_controller.rb b/app/controllers/import_zendesk_controller.rb new file mode 100644 index 000000000..0cffb6557 --- /dev/null +++ b/app/controllers/import_zendesk_controller.rb @@ -0,0 +1,124 @@ +# Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/ +require 'zendesk_api' + +class ImportZendeskController < ApplicationController + + def url_check + return if setup_done_response + + # validate + if !params[:url] || params[:url] !~ %r{^(http|https)://.+?$} + render json: { + result: 'invalid', + message: 'Invalid URL!', + } + return + end + + # connection test + translation_map = { + 'No such file' => 'Hostname not found!', + 'getaddrinfo: nodename nor servname provided, or not known' => 'Hostname not found!', + 'No route to host' => 'No route to host!', + 'Connection refused' => 'Connection refused!', + } + + response = UserAgent.request( params[:url] ) + + if !response.success? + message_human = '' + translation_map.each {|key, message| + if response.error.to_s =~ /#{Regexp.escape(key)}/i + message_human = message + end + } + render json: { + result: 'invalid', + message_human: message_human, + message: response.error.to_s, + } + return + end + + Setting.set('import_zendesk_endpoint', "#{params[:url]}api/v2") + + render json: { + result: 'ok', + url: params[:url], + } + end + + def credentials_check + return if setup_done_response + + if !params[:username] || !params[:token] + + render json: { + result: 'invalid', + message_human: 'Incomplete credentials', + } + return + end + + Setting.set('import_zendesk_endpoint_username', params[:username]) + Setting.set('import_zendesk_endpoint_key', params[:token]) + + if !Import::Zendesk.connection_test + + Setting.set('import_zendesk_endpoint_username', nil) + Setting.set('import_zendesk_endpoint_key', nil) + + render json: { + result: 'invalid', + message_human: 'Invalid credentials!', + } + return + end + + render json: { + result: 'ok', + } + end + + def import_start + return if setup_done_response + Setting.set('import_mode', true) + + # start migration + Import::Zendesk.delay.start_bg + + render json: { + result: 'ok', + } + end + + def import_status + result = Import::Zendesk.status_bg + if result[:result] == 'import_done' + Setting.reload + end + render json: result + end + + private + + def setup_done + count = User.all.count() + done = true + if count <= 2 + done = false + end + done + end + + def setup_done_response + if !setup_done + return false + end + render json: { + setup_done: true, + } + true + end + +end diff --git a/config/routes/import_zendesk.rb b/config/routes/import_zendesk.rb new file mode 100644 index 000000000..4d1da3f59 --- /dev/null +++ b/config/routes/import_zendesk.rb @@ -0,0 +1,10 @@ +Zammad::Application.routes.draw do + api_path = Rails.configuration.api_path + + # import zendesk + match api_path + '/import/zendesk/url_check', to: 'import_zendesk#url_check', via: :post + match api_path + '/import/zendesk/credentials_check', to: 'import_zendesk#credentials_check', via: :post + match api_path + '/import/zendesk/import_start', to: 'import_zendesk#import_start', via: :post + match api_path + '/import/zendesk/import_status', to: 'import_zendesk#import_status', via: :get + +end diff --git a/lib/import/zendesk.rb b/lib/import/zendesk.rb index e04326d8d..1b8d783a6 100644 --- a/lib/import/zendesk.rb +++ b/lib/import/zendesk.rb @@ -48,6 +48,95 @@ module Import::Zendesk true end +=begin + start import in background + + Import::Zendesk.start_bg +=end + + def start_bg + Setting.reload + + Import::Zendesk.connection_test + + # start thread to observe current state + status_update_thread = Thread.new { + loop do + result = { + data: current_state, + result: 'in_progress', + } + Cache.write('import:state', result, expires_in: 10.minutes) + sleep 8 + end + } + sleep 2 + + # start thread to import data + begin + import_thread = Thread.new { + Import::Zendesk.start + } + rescue => e + status_update_thread.exit + status_update_thread.join + Rails.logger.error e.message + Rails.logger.error e.backtrace.inspect + result = { + message: e.message, + result: 'error', + } + Cache.write('import:state', result, expires_in: 10.hours) + return false + end + import_thread.join + status_update_thread.exit + status_update_thread.join + + result = { + result: 'import_done', + } + Cache.write('import:state', result, expires_in: 10.hours) + + Setting.set('system_init_done', true) + Setting.set('import_mode', false) + end + +=begin + + get import state from background process + + result = Import::Zendesk.status_bg + +=end + + def status_bg + state = Cache.get('import:state') + return state if state + { + message: 'not running', + } + end + +=begin + + start get request to backend to check connection + + result = connection_test + + return + + true | false + +=end + + def connection_test + initialize_client + + return true if @client.users.first + false + end + def statistic # check cache @@ -89,9 +178,61 @@ module Import::Zendesk statistic end - def initialize_client - return nil if @client +=begin + return current import state + + result = current_state + + return + + { + :Group => { + :total => 1234, + :done => 13, + }, + :Organization => { + :total => 1234, + :done => 13, + }, + :User => { + :total => 1234, + :done => 13, + }, + :Ticket => { + :total => 1234, + :done => 13, + }, + } + +=end + + def current_state + + data = statistic + + # TODO: Ticket, User, Organization fields + { + Group: { + done: Group.count, + total: data['Groups'] || 0, + }, + Organization: { + done: Organization.count, + total: data['Organizations'] || 0, + }, + User: { + done: User.count, + total: data['Users'] || 0, + }, + Ticket: { + done: Ticket.count, + total: data['Tickets'] || 0, + }, + } + end + + def initialize_client @client = ZendeskAPI::Client.new do |config| config.url = Setting.get('import_zendesk_endpoint') @@ -350,11 +491,13 @@ module Import::Zendesk @client.organizations.each { |zendesk_organization| local_organization_fields = { - name: zendesk_organization.name, - note: zendesk_organization.note, - shared: zendesk_organization.shared_tickets, + name: zendesk_organization.name, + note: zendesk_organization.note, + shared: zendesk_organization.shared_tickets, # shared: zendesk_organization.shared_comments, # TODO, not yet implemented # }.merge(zendesk_organization.organization_fields) # TODO + updated_by_id: 1, + created_by_id: 1 } local_organization = Organization.create_if_not_exists( local_organization_fields ) @@ -391,6 +534,8 @@ module Import::Zendesk verified: zendesk_user.verified, organization_id: @zendesk_organization_mapping[ zendesk_user.organization_id ], last_login: zendesk_user.last_login_at, + updated_by_id: 1, + created_by_id: 1 } if @zendesk_user_group_mapping[ zendesk_user.id ] @@ -498,19 +643,18 @@ module Import::Zendesk title: zendesk_ticket.subject, note: zendesk_ticket.description, group_id: @zendesk_group_mapping[ zendesk_ticket.group_id ] || 1, - customer_id: @zendesk_user_mapping[ zendesk_ticket.requester_id ], + customer_id: @zendesk_user_mapping[ zendesk_ticket.requester_id ] || 1, organization_id: @zendesk_organization_mapping[ zendesk_ticket.organization_id ], state: Ticket::State.lookup( name: mapping_state( zendesk_ticket.status ) ), priority: Ticket::Priority.lookup( name: mapping_priority( zendesk_ticket.priority ) ), pending_time: zendesk_ticket.due_at, updated_at: zendesk_ticket.updated_at, created_at: zendesk_ticket.created_at, - updated_by_id: @zendesk_user_mapping[ zendesk_ticket.requester_id ], - created_by_id: @zendesk_user_mapping[ zendesk_ticket.requester_id ], + updated_by_id: @zendesk_user_mapping[ zendesk_ticket.requester_id ] || 1, + created_by_id: @zendesk_user_mapping[ zendesk_ticket.requester_id ] || 1, # }.merge(zendesk_ticket_fields) TODO } - - ticket_author = User.find( @zendesk_user_mapping[ zendesk_ticket.requester_id ] ) + ticket_author = User.find( @zendesk_user_mapping[ zendesk_ticket.requester_id ] || 1 ) local_ticket_fields[:create_article_sender_id] = if ticket_author.role?('Customer') article_sender_customer.id @@ -553,7 +697,7 @@ module Import::Zendesk object: 'Ticket', o_id: local_ticket.id, item: tag.id, - created_by_id: @zendesk_user_mapping[ zendesk_ticket.requester_id ], + created_by_id: @zendesk_user_mapping[ zendesk_ticket.requester_id ] || 1, ) } @@ -581,11 +725,11 @@ module Import::Zendesk ticket_id: local_ticket.id, body: zendesk_article.html_body, internal: !zendesk_article.public, - updated_by_id: @zendesk_user_mapping[ zendesk_article.author_id ], - created_by_id: @zendesk_user_mapping[ zendesk_article.author_id ], + updated_by_id: @zendesk_user_mapping[ zendesk_article.author_id ] || 1, + created_by_id: @zendesk_user_mapping[ zendesk_article.author_id ] || 1, } - article_author = User.find( @zendesk_user_mapping[ zendesk_article.author_id ] ) + article_author = User.find( @zendesk_user_mapping[ zendesk_article.author_id ] || 1 ) local_article_fields[:sender_id] = if article_author.role?('Customer') article_sender_customer.id @@ -662,7 +806,8 @@ module Import::Zendesk filename: zendesk_attachment.file_name, preferences: { 'Content-Type' => zendesk_attachment.content_type - } + }, + created_by_id: 1 ) } } diff --git a/test/integration/zendesk_import_browser_test.rb b/test/integration/zendesk_import_browser_test.rb new file mode 100644 index 000000000..70b414d4e --- /dev/null +++ b/test/integration/zendesk_import_browser_test.rb @@ -0,0 +1,87 @@ +# encoding: utf-8 +require 'browser_test_helper' + +class ZendeskImportBrowserTest < TestCase + def test_import + + if !ENV['IMPORT_BT_ZENDESK_ENDPOINT'] + fail "ERROR: Need IMPORT_BT_ZENDESK_ENDPOINT - hint IMPORT_BT_ZENDESK_ENDPOINT='https://example.zendesk.com/' (including trailing slash!)" + end + if !ENV['IMPORT_BT_ZENDESK_ENDPOINT_USERNAME'] + fail "ERROR: Need IMPORT_BT_ZENDESK_ENDPOINT_USERNAME - hint IMPORT_BT_ZENDESK_ENDPOINT_USERNAME='your@email.com'" + end + if !ENV['IMPORT_BT_ZENDESK_ENDPOINT_KEY'] + fail "ERROR: Need IMPORT_BT_ZENDESK_ENDPOINT_KEY - hint IMPORT_BT_ZENDESK_ENDPOINT_KEY='XYZ3133723421111'" + end + + @browser = browser_instance + location(url: browser_url) + + click(css: 'a[href="#import"]') + + click(css: 'a[href="#import/zendesk"]') + + set( + css: '#zendesk-url', + value: 'https://reallybadexample.zendesk.com/' + ) + + sleep 5 + + watch_for( + css: '.zendesk-url-error', + value: 'Hostname not found!', + ) + + set( + css: '#zendesk-url', + value: ENV['IMPORT_BT_ZENDESK_ENDPOINT'] + ) + + sleep 5 + + watch_for_disappear( + css: '.zendesk-url-error', + value: 'Hostname not found!', + ) + + click(css: '.js-zendesk-credentials') + + set( + css: '#zendesk-email', + value: ENV['IMPORT_BT_ZENDESK_ENDPOINT_USERNAME'] + ) + + set( + css: '#zendesk-api-token', + value: '1nv4l1dT0K3N' + ) + + sleep 5 + + watch_for( + css: '.zendesk-api-token-error', + value: 'Invalid credentials!', + ) + + set( + css: '#zendesk-api-token', + value: ENV['IMPORT_BT_ZENDESK_ENDPOINT_KEY'] + ) + + sleep 5 + + watch_for_disappear( + css: '.zendesk-url-error', + value: 'Invalid credentials!', + ) + + click(css: '.js-migration-start') + + watch_for( + css: 'body', + value: 'login', + timeout: 300, + ) + end +end