From 781ffa3209cb39e4a7f7d8ab64390718003b37b3 Mon Sep 17 00:00:00 2001 From: Denny Bresch Date: Mon, 12 Aug 2019 17:07:06 +0200 Subject: [PATCH] Maintenance: Improve handling of Zendesk API to get the correct count of objects while import. --- .../zendesk/mixin/incremental_export.rb | 62 +++++++++++++++++++ .../import/zendesk/objects_total_count.rb | 38 +++++------- .../unit/import/zendesk/organizations.rb | 6 -- lib/sequencer/unit/import/zendesk/tickets.rb | 56 ----------------- 4 files changed, 76 insertions(+), 86 deletions(-) diff --git a/lib/sequencer/unit/import/zendesk/mixin/incremental_export.rb b/lib/sequencer/unit/import/zendesk/mixin/incremental_export.rb index c9392cda6..ccbef2c51 100644 --- a/lib/sequencer/unit/import/zendesk/mixin/incremental_export.rb +++ b/lib/sequencer/unit/import/zendesk/mixin/incremental_export.rb @@ -5,6 +5,8 @@ class Sequencer module Mixin module IncrementalExport + attr_accessor :previous_page + def self.included(base) base.uses :client end @@ -12,6 +14,66 @@ class Sequencer def resource_collection "::ZendeskAPI::#{resource_klass}".constantize.incremental_export(client, 1) end + + def resource_iteration + super do |record| + # call passed/originally intended block + yield(record) + + # add hook to check if object (e.g. ticket) count + # update is needed because the request + # might have changed + update_count + end + end + + # The source if this is the limitation of not knowing + # how much objects there are in total before requesting the endpoint + # This is caused by the Zendesk API which only returns max. 1000 + # per request + def update_count + update_import_job + self.previous_page = next_page + end + + def update_import_job + return if !update_required? + + state.provide(:import_job, updated_import_job) + end + + def klass_key + resource_klass.singularize.to_sym + end + + def updated_import_job + + import_job.result[klass_key].merge( + total: import_job.result[klass_key][:total] + current_request_count + ) + import_job + end + + def update_required? + return false if previous_page.blank? + return false if previous_page == next_page + + current_request_count.present? + end + + def current_request_count + # access the internal instance method of the + # Zendesk collection request to get the current + # count of the endpoint (max. 1000) + resource_collection_attribute.instance_variable_get(:@count) + end + + def next_page + # access the internal instance method of the + # Zendesk collection request to get the next + # page number of the endpoint + resource_collection_attribute.instance_variable_get(:@next_page) + end end end end diff --git a/lib/sequencer/unit/import/zendesk/objects_total_count.rb b/lib/sequencer/unit/import/zendesk/objects_total_count.rb index ebbd68824..cb0facae9 100644 --- a/lib/sequencer/unit/import/zendesk/objects_total_count.rb +++ b/lib/sequencer/unit/import/zendesk/objects_total_count.rb @@ -17,32 +17,22 @@ class Sequencer end end - def request(object) - return tickets if object == 'Tickets' - - generic(object) - end - - def generic(object) - client.send(object.to_s.underscore.to_sym) - end - - # this special ticket logic is needed since Zendesk archives tickets - # after 120 days and doesn't return them via the client.tickets + # the special "incremental_export" logic is needed because Zendesk + # archives records and doesn't return them via e.g. client.tickets # endpoint as described here: # https://github.com/zammad/zammad/issues/558#issuecomment-267951351 - # the proper way is to use the 'incremental' endpoint which is not available - # via the ruby gem yet but a pull request is pending: - # https://github.com/zendesk/zendesk_api_client_rb/pull/287 - # the following workaround is needed to use this functionality - # Counting Tickets has the limitations that max. 1000 are returned - # that's why we need to update the number when it's exceeded while importing - def tickets - ZendeskAPI::Collection.new( - client, - ZendeskAPI::Ticket, - path: 'incremental/tickets?start_time=1' - ) + # Counting via the incremental_export endpoint has the limitations + # that it returns max. 1000. That's why we need to update the total + # number while importing in the resource loop + def request(object) + resource_class = "::ZendeskAPI::#{object.singularize}".safe_constantize + if resource_class.respond_to?(:incremental_export) + # read as: ::ZendeskAPI::Ticket.incremental_export(client, 1) + resource_class.incremental_export(client, 1) + else + # read as: client.groups + client.send(object.to_s.underscore.to_sym) + end end end end diff --git a/lib/sequencer/unit/import/zendesk/organizations.rb b/lib/sequencer/unit/import/zendesk/organizations.rb index b74d46993..41c3a1879 100644 --- a/lib/sequencer/unit/import/zendesk/organizations.rb +++ b/lib/sequencer/unit/import/zendesk/organizations.rb @@ -4,12 +4,6 @@ class Sequencer module Zendesk class Organizations < Sequencer::Unit::Import::Zendesk::SubSequence::Object include ::Sequencer::Unit::Import::Zendesk::Mixin::IncrementalExport - - private - - def resource_iteration_method - :all! - end end end end diff --git a/lib/sequencer/unit/import/zendesk/tickets.rb b/lib/sequencer/unit/import/zendesk/tickets.rb index d4ecfc801..a6f12788d 100644 --- a/lib/sequencer/unit/import/zendesk/tickets.rb +++ b/lib/sequencer/unit/import/zendesk/tickets.rb @@ -17,62 +17,6 @@ class Sequencer ticket_field_map: ticket_field_map, ) end - - def resource_iteration - super do |record| - # call passed/originally intended block - yield(record) - - # add hook to check if ticket count - # update is needed because the request - # might have changed - update_ticket_count - end - end - - # The source if this is the limitation of not knowing - # how much tickets there are in total before requesting the endpoint - # This is caused by the Zendesk API which only returns max. 1000 - # per request - def update_ticket_count - update_import_job - next_page - end - - attr_accessor :previous_page - - def update_import_job - return if !update_required? - - state.provide(import_job, updated_import_job) - end - - def updated_import_job - import_job.result[:Tickets].merge( - total: import_job.result[:Tickets][:total] + current_request_count - ) - end - - def update_required? - return false if previous_page.blank? - return false if previous_page == next_page - - current_request_count.present? - end - - def current_request_count - # access the internal instance method of the - # Zendesk collection request to get the current - # count of the endpoint (max. 1000) - resource_collection_attribute.instance_variable_get(:@count) - end - - def next_page - # access the internal instance method of the - # Zendesk collection request to get the next - # page number of the endpoint - resource_collection_attribute.instance_variable_get(:@next_page) - end end end end