Fixes #3631, fixes #3645 - Freshdesk import subdomain selection fails.

app/controllers/import_freshdesk_controller.rb

@@ -14,9 +14,12 @@ class ImportFreshdeskController < ApplicationController
       return
     end
 
-    response = UserAgent.request(params[:url])
+    endpoint = "#{params[:url]}/api/v2"
+    endpoint.gsub!(%r{([^:])//+}, '\\1/')
 
-    if !response.success?
+    response = UserAgent.request("#{endpoint}/contacts")
+
+    if response.header.nil? || !response.header['x-freshdesk-api-version']
       render json: {
         result:        'invalid',
         message_human: url_check_human_error_message(response.error.to_s),
@@ -25,18 +28,6 @@ class ImportFreshdeskController < ApplicationController
       return
     end
 
-    # Check if maybe a redirect is implemented.
-    if !response.body.match?(%r{#{params[:url]}})
-      render json: {
-        result:        'invalid',
-        message_human: 'Hostname not found!',
-      }
-      return
-    end
-
-    endpoint = "#{params[:url]}/api/v2"
-    endpoint.gsub!(%r{([^:])//+}, '\\1/')
-
     Setting.set('import_freshdesk_endpoint', endpoint)
 
     render json: {

lib/user_agent.rb

@@ -453,19 +453,22 @@ returns
         error:   "No such file #{uri}, 404!",
         success: false,
         code:    response.code,
+        header:  response.each_header.to_h,
       )
     when Net::HTTPClientError
       return Result.new(
         error:   "Client Error: #{response.inspect}!",
         success: false,
         code:    response.code,
-        body:    response.body
+        body:    response.body,
+        header:  response.each_header.to_h,
       )
     when Net::HTTPInternalServerError
       return Result.new(
         error:   "Server Error: #{response.inspect}!",
         success: false,
         code:    response.code,
+        header:  response.each_header.to_h,
       )
     when Net::HTTPRedirection
       raise 'Too many redirections for the original URL, halting.' if count <= 0
@@ -483,6 +486,7 @@ returns
         content_type: response['Content-Type'],
         success:      true,
         code:         response.code,
+        header:       response.each_header.to_h,
       )
     end
 
@@ -546,7 +550,7 @@ returns
 
   class Result
 
-    attr_reader :error, :body, :data, :code, :content_type
+    attr_reader :error, :body, :data, :code, :content_type, :header
 
     def initialize(options)
       @success      = options[:success]
@@ -555,6 +559,7 @@ returns
       @code         = options[:code]
       @content_type = options[:content_type]
       @error        = options[:error]
+      @header       = options[:header]
     end
 
     def success?

spec/requests/import_freshdesk_spec.rb

@@ -0,0 +1,27 @@
+# Copyright (C) 2012-2021 Zammad Foundation, http://zammad-foundation.org/
+
+require 'rails_helper'
+
+RSpec.describe 'ImportFreshdesk', type: :request, set_up: false, authenticated_as: false, required_envs: %w[IMPORT_FRESHDESK_ENDPOINT_SUBDOMAIN] do
+  let(:action) { nil }
+  let(:endpoint) { "/api/v1/import/freshdesk/#{action}" }
+
+  describe 'POST /api/v1/import/freshdesk/url_check', :use_vcr do
+    let(:action) { 'url_check' }
+
+    it 'check invalid subdomain' do
+      post endpoint, params: { url: 'https://reallybadexample.freshdesk.com' }, as: :json
+      expect(json_response['result']).to eq('invalid')
+    end
+
+    it 'check valid subdomain' do
+      post endpoint, params: { url: "https://#{ENV['IMPORT_FRESHDESK_ENDPOINT_SUBDOMAIN']}.freshdesk.com" }, as: :json
+      expect(json_response['result']).to eq('ok')
+    end
+
+    it 'check valid subdomain with special configuration (e.g. SSO / OAUTH)' do
+      post endpoint, params: { url: 'https://example.freshdesk.com' }, as: :json
+      expect(json_response['result']).to eq('ok')
+    end
+  end
+end

test/data/vcr_cassettes/requests/import_freshdesk/importfreshdesk_post_api_v1_import_freshdesk_url_check_check_invalid_subdomain.yml

@@ -0,0 +1,57 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://reallybadexample.freshdesk.com/api/v2/contacts
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Zammad User Agent
+      Host:
+      - reallybadexample.freshdesk.com
+  response:
+    status:
+      code: 404
+      message: Not Found
+    headers:
+      Date:
+      - Fri, 02 Jul 2021 13:50:52 GMT
+      Content-Type:
+      - application/json; charset=utf-8
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Status:
+      - 404 Not Found
+      X-Request-Id:
+      - a2b52e91-304f-45d0-9a9f-c561bd4ab7b6
+      X-Rack-Cache:
+      - miss
+      Cache-Control:
+      - no-cache
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Ua-Compatible:
+      - IE=Edge,chrome=1
+      X-Content-Type-Options:
+      - nosniff
+      X-Fw-Ratelimiting-Managed:
+      - 'false'
+      X-Envoy-Upstream-Service-Time:
+      - '34'
+      X-Trace-Id:
+      - 00-b5935665b002c157f9507abedbfddb3c-484e6fec463479ed-00
+      Server:
+      - fwe
+    body:
+      encoding: ASCII-8BIT
+      string: " "
+  recorded_at: Fri, 02 Jul 2021 13:50:52 GMT
+recorded_with: VCR 6.0.0

test/data/vcr_cassettes/requests/import_freshdesk/importfreshdesk_post_api_v1_import_freshdesk_url_check_check_valid_subdomain.yml

@@ -0,0 +1,68 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://<IMPORT_FRESHDESK_ENDPOINT_SUBDOMAIN>.freshdesk.com/api/v2/contacts
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Zammad User Agent
+      Host:
+      - "<IMPORT_FRESHDESK_ENDPOINT_SUBDOMAIN>.freshdesk.com"
+  response:
+    status:
+      code: 401
+      message: Unauthorized
+    headers:
+      Date:
+      - Fri, 02 Jul 2021 13:50:52 GMT
+      Content-Type:
+      - application/json; charset=utf-8
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Status:
+      - 401 Unauthorized
+      X-Request-Id:
+      - adecf6f7-532b-4a6b-b28f-22c2dbb72818
+      X-Freshdesk-Api-Version:
+      - latest=v2; requested=v2
+      X-Rack-Cache:
+      - miss
+      Cache-Control:
+      - no-cache
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Ua-Compatible:
+      - IE=Edge,chrome=1
+      X-Content-Type-Options:
+      - nosniff
+      Set-Cookie:
+      - _x_w=5_2; path=/; HttpOnly; secure
+      X-Fw-Ratelimiting-Managed:
+      - 'true'
+      X-Ratelimit-Total:
+      - '40'
+      X-Ratelimit-Remaining:
+      - '39'
+      X-Ratelimit-Used-Currentrequest:
+      - '1'
+      X-Envoy-Upstream-Service-Time:
+      - '35'
+      X-Trace-Id:
+      - 00-2737df9995dcf3e1e34c1ee5b8a525f7-44657fcc6d9a6433-00
+      Server:
+      - fwe
+    body:
+      encoding: UTF-8
+      string: '{"code":"invalid_credentials","message":"You have to be logged in to
+        perform this action."}'
+  recorded_at: Fri, 02 Jul 2021 13:50:52 GMT
+recorded_with: VCR 6.0.0

test/data/vcr_cassettes/requests/import_freshdesk/importfreshdesk_post_api_v1_import_freshdesk_url_check_check_valid_subdomain_with_special_configuration_e_g_sso_oauth_.yml

@@ -0,0 +1,69 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://example.freshdesk.com/api/v2/contacts
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Zammad User Agent
+      Host:
+      - example.freshdesk.com
+  response:
+    status:
+      code: 401
+      message: Unauthorized
+    headers:
+      Date:
+      - Fri, 02 Jul 2021 13:50:52 GMT
+      Content-Type:
+      - application/json; charset=utf-8
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Status:
+      - 401 Unauthorized
+      X-Ratelimit-Remaining:
+      - '4998'
+      X-Ratelimit-Total:
+      - '5000'
+      X-Request-Id:
+      - f252fac3-343c-4f56-9306-56012f116d74
+      X-Freshdesk-Api-Version:
+      - latest=v2; requested=v2
+      X-Rack-Cache:
+      - miss
+      Cache-Control:
+      - no-cache
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Ratelimit-Used-Currentrequest:
+      - '1'
+      X-Ua-Compatible:
+      - IE=Edge,chrome=1
+      X-Content-Type-Options:
+      - nosniff
+      Set-Cookie:
+      - _x_m=x_a; path=/; HttpOnly; secure
+      - _x_w=39_1; path=/; HttpOnly; secure
+      X-Fw-Ratelimiting-Managed:
+      - 'false'
+      X-Envoy-Upstream-Service-Time:
+      - '58'
+      X-Trace-Id:
+      - 00-f37a5fc3bc948c193eb81c7049f6fd66-135d0d9947976cfe-00
+      Server:
+      - fwe
+    body:
+      encoding: UTF-8
+      string: '{"code":"invalid_credentials","message":"You have to be logged in to
+        perform this action."}'
+  recorded_at: Fri, 02 Jul 2021 13:50:52 GMT
+recorded_with: VCR 6.0.0

test/data/vcr_cassettes/system/import/freshdesk/import_freshdesk_fields_validation_invalid_hostname.yml

@@ -2,7 +2,7 @@
 http_interactions:
 - request:
     method: get
-    uri: https://reallybadexample.freshdesk.com/
+    uri: https://reallybadexample.freshdesk.com/api/v2/contacts
     body:
       encoding: US-ASCII
       string: ''
@@ -17,56 +17,41 @@ http_interactions:
       - reallybadexample.freshdesk.com
   response:
     status:
-      code: 200
+      code: 404
-      message: OK
+      message: Not Found
     headers:
       Date:
-      - Wed, 30 Jun 2021 09:10:23 GMT
+      - Fri, 02 Jul 2021 17:06:16 GMT
       Content-Type:
-      - text/html; charset=utf-8
+      - application/json; charset=utf-8
       Transfer-Encoding:
       - chunked
       Connection:
       - keep-alive
       Status:
-      - 200 OK
+      - 404 Not Found
-      Cache-Control:
-      - max-age=0, private, must-revalidate
-      X-Xss-Protection:
-      - 1; mode=block
       X-Request-Id:
-      - 60005b71-aa9e-4615-b5e6-1787b42e890a
+      - 200f4a46-412a-4e8a-b417-2639ef0c7dfe
-      X-Ua-Compatible:
-      - IE=Edge,chrome=1
-      Etag:
-      - W/"1f9efcb8395f73f1b213752ae9941ad3"
-      X-Frame-Options:
-      - SAMEORIGIN
-      X-Content-Type-Options:
-      - nosniff
       X-Rack-Cache:
       - miss
+      Cache-Control:
+      - no-cache
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Ua-Compatible:
+      - IE=Edge,chrome=1
+      X-Content-Type-Options:
+      - nosniff
+      X-Fw-Ratelimiting-Managed:
+      - 'false'
       X-Envoy-Upstream-Service-Time:
-      - '27'
+      - '902'
       X-Trace-Id:
-      - 00-35567a8fbfd7346dc95ea2a8b0c9320b-f68d5ac26a465810-00
+      - 00-616a357299270485862d4b040120e5ef-32eb6e8fbf69042e-00
       Server:
       - fwe
     body:
       encoding: ASCII-8BIT
-      string: "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n<html
+      string: " "
-        xmlns=\"http://www.w3.org/1999/xhtml\">\n\t<head>\n\t\t<meta http-equiv=\"content-type\"
+  recorded_at: Fri, 02 Jul 2021 17:06:16 GMT
-        content=\"text/html; charset=utf-8\" />\n\t\t<meta name=\"robots\" content=\"noindex\"/>\n\t\t<meta
-        name=\"robots\" content=\"noarchive\"/>\n\t\t<link rel=\"stylesheet\" href=\"/assets/page/404.css\"
-        type=\"text/css\" media=\"screen, projection\" />\n\t\t<title>There is no
-        helpdesk here!</title>\n\t</head>\n\t<body>\n\t\t<div id=\"wrap\" class=\"container\">\n\t\t\t<div
-        class=\"wrapper\">\n\t\t\t\t<div class=\"wrapper-shadow\">\n\t\t\t\t\t<div
-        class=\"no-helpdesk\">\n\t\t\t\t\t\t<img src=\"/assets/misc/freshdesklogo.png\"
-        alt=\"Freshdesk\" width=\"204px\" height=\"50px\" />\n\t\t\t\t\t\t<h2>  We
-        couldn&#x27;t find  <span id=\"domainname\"></span></h2>\n\t\t\t\t\t\t<p>
-        May be this is still fresh!  </p> \n\t\t\t\t\t\t<p> You can claim it now at
-        <a href=\"http://www.freshdesk.com/signup\">http://www.freshdesk.com/signup</a></p>\n\t\t\t\t\t</div>\n\t\t\t\t</div>\n\t\t\t</div>\n\t\t</div>\n\t\t<script
-        type=\"text/javascript\">\n//<![CDATA[\n\n\t\t\tdocument.getElementById(\"domainname\").innerHTML
-        = (document.domain);\n\n//]]>\n</script>\t</body>\n</html>\n"
-  recorded_at: Wed, 30 Jun 2021 09:10:23 GMT
 recorded_with: VCR 6.0.0

test/data/vcr_cassettes/system/import/freshdesk/import_freshdesk_fields_validation_valid_hostname.yml

@@ -2,7 +2,7 @@
 http_interactions:
 - request:
     method: get
-    uri: https://reallybadexample.freshdesk.com/
+    uri: https://reallybadexample.freshdesk.com/api/v2/contacts
     body:
       encoding: US-ASCII
       string: ''
@@ -17,56 +17,41 @@ http_interactions:
       - reallybadexample.freshdesk.com
   response:
     status:
-      code: 200
+      code: 404
-      message: OK
+      message: Not Found
     headers:
       Date:
-      - Wed, 30 Jun 2021 09:10:25 GMT
+      - Fri, 02 Jul 2021 17:06:18 GMT
       Content-Type:
-      - text/html; charset=utf-8
+      - application/json; charset=utf-8
       Transfer-Encoding:
       - chunked
       Connection:
       - keep-alive
       Status:
-      - 200 OK
+      - 404 Not Found
-      Cache-Control:
-      - max-age=0, private, must-revalidate
-      X-Xss-Protection:
-      - 1; mode=block
       X-Request-Id:
-      - decbd7ab-516e-489c-b22f-ee24fb52e8eb
+      - ea835512-a5f9-4872-bcf0-102168c453a6
-      X-Ua-Compatible:
-      - IE=Edge,chrome=1
-      Etag:
-      - W/"1f9efcb8395f73f1b213752ae9941ad3"
-      X-Frame-Options:
-      - SAMEORIGIN
-      X-Content-Type-Options:
-      - nosniff
       X-Rack-Cache:
       - miss
+      Cache-Control:
+      - no-cache
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Ua-Compatible:
+      - IE=Edge,chrome=1
+      X-Content-Type-Options:
+      - nosniff
+      X-Fw-Ratelimiting-Managed:
+      - 'false'
       X-Envoy-Upstream-Service-Time:
-      - '22'
+      - '31'
       X-Trace-Id:
-      - 00-9f384d36e91558f80aa90fd756df07d1-bfa5e51d6fac7500-00
+      - 00-4aeca9c6614c654f313a692679670477-54a60f5fbd984c56-00
       Server:
       - fwe
     body:
       encoding: ASCII-8BIT
-      string: "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n<html
+      string: " "
-        xmlns=\"http://www.w3.org/1999/xhtml\">\n\t<head>\n\t\t<meta http-equiv=\"content-type\"
+  recorded_at: Fri, 02 Jul 2021 17:06:18 GMT
-        content=\"text/html; charset=utf-8\" />\n\t\t<meta name=\"robots\" content=\"noindex\"/>\n\t\t<meta
-        name=\"robots\" content=\"noarchive\"/>\n\t\t<link rel=\"stylesheet\" href=\"/assets/page/404.css\"
-        type=\"text/css\" media=\"screen, projection\" />\n\t\t<title>There is no
-        helpdesk here!</title>\n\t</head>\n\t<body>\n\t\t<div id=\"wrap\" class=\"container\">\n\t\t\t<div
-        class=\"wrapper\">\n\t\t\t\t<div class=\"wrapper-shadow\">\n\t\t\t\t\t<div
-        class=\"no-helpdesk\">\n\t\t\t\t\t\t<img src=\"/assets/misc/freshdesklogo.png\"
-        alt=\"Freshdesk\" width=\"204px\" height=\"50px\" />\n\t\t\t\t\t\t<h2>  We
-        couldn&#x27;t find  <span id=\"domainname\"></span></h2>\n\t\t\t\t\t\t<p>
-        May be this is still fresh!  </p> \n\t\t\t\t\t\t<p> You can claim it now at
-        <a href=\"http://www.freshdesk.com/signup\">http://www.freshdesk.com/signup</a></p>\n\t\t\t\t\t</div>\n\t\t\t\t</div>\n\t\t\t</div>\n\t\t</div>\n\t\t<script
-        type=\"text/javascript\">\n//<![CDATA[\n\n\t\t\tdocument.getElementById(\"domainname\").innerHTML
-        = (document.domain);\n\n//]]>\n</script>\t</body>\n</html>\n"
-  recorded_at: Wed, 30 Jun 2021 09:10:26 GMT
 recorded_with: VCR 6.0.0

test/data/vcr_cassettes/system/import/freshdesk/import_freshdesk_import_progress_shows_organizations_progress.yml

@@ -0,0 +1,78 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://<IMPORT_FRESHDESK_ENDPOINT_SUBDOMAIN>.freshdesk.com/api/v2/agents/me
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Content-Type:
+      - application/json
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+      Host:
+      - "<IMPORT_FRESHDESK_ENDPOINT_SUBDOMAIN>.freshdesk.com"
+      Authorization:
+      - Basic MWlPRmFiVjFYc1kwZ3h0eU9ud1E6WA==
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Date:
+      - Sat, 03 Jul 2021 06:18:11 GMT
+      Content-Type:
+      - application/json; charset=utf-8
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Status:
+      - 200 OK
+      Pragma:
+      - no-cache
+      X-Request-Id:
+      - 4e07257c-5b3a-4a2b-8c32-17f7088aa701
+      X-Freshdesk-Api-Version:
+      - latest=v2; requested=v2
+      X-Rack-Cache:
+      - miss
+      Cache-Control:
+      - must-revalidate, no-cache, no-store, private, max-age=0
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Ua-Compatible:
+      - IE=Edge,chrome=1
+      X-Content-Type-Options:
+      - nosniff
+      Expires:
+      - Wed, 13 Oct 2010 00:00:00 UTC
+      Set-Cookie:
+      - _helpkit_session=BAh7B0kiD3Nlc3Npb25faWQGOgZFVEkiJWRiNWJjYjE2MjJhNjYzODY5NTZjMDY1OWEwOTI0ODZjBjsAVEkiEF9jc3JmX3Rva2VuBjsARkkiMUJ1clZXWU42c0pQb21BdytWQUFWczVwb2pHZXdvckNYQVZuTEVFaDE0U0E9BjsARg%3D%3D--98cd298564a0bfe7396d38be08f58ed98127adad;
+        path=/; HttpOnly; secure
+      - _x_w=5_2; path=/; HttpOnly; secure
+      X-Fw-Ratelimiting-Managed:
+      - 'true'
+      X-Ratelimit-Total:
+      - '100'
+      X-Ratelimit-Remaining:
+      - '99'
+      X-Ratelimit-Used-Currentrequest:
+      - '1'
+      X-Envoy-Upstream-Service-Time:
+      - '45'
+      X-Trace-Id:
+      - 00-f906ee496f5f50e1c8766cb5323ad55f-2a5d9f13ea92af0d-00
+      Server:
+      - fwe
+    body:
+      encoding: ASCII-8BIT
+      string: '{"available":false,"occasional":false,"id":80014400475,"ticket_scope":1,"signature":null,"group_ids":[],"role_ids":[80000198826],"skill_ids":[],"available_since":null,"contact":{"active":true,"email":"info@<IMPORT_FRESHDESK_ENDPOINT_SUBDOMAIN>.org","job_title":null,"language":"en","mobile":null,"name":"Thorsten
+        Eckel","phone":null,"time_zone":"Eastern Time (US & Canada)","created_at":"2021-04-09T13:23:58Z","updated_at":"2021-04-09T13:31:00Z","last_login_at":"2021-06-04T06:55:46Z"},"created_at":"2021-04-09T13:23:58Z","updated_at":"2021-06-08T09:56:50Z","type":"support_agent"}'
+  recorded_at: Sat, 03 Jul 2021 06:18:12 GMT
+recorded_with: VCR 6.0.0

test/integration/user_agent_test.rb

@@ -25,6 +25,8 @@ class UserAgentTest < ActiveSupport::TestCase
     assert_equal(true, result.success?)
     assert_equal('200', result.code)
     assert_equal(String, result.body.class)
+    assert_equal(Hash, result.header.class)
+    assert_equal('application/json; charset=utf-8', result.header['content-type'])
     assert(result.body.include?('"get"'))
     assert(result.body.include?('"123"'))
     assert(result.body.include?('"content_type_requested":null'))