Maintenance: Improve handling of Zendesk API to get the correct count of objects while import.

lib/sequencer/unit/import/zendesk/mixin/incremental_export.rb

@@ -5,6 +5,8 @@ class Sequencer
         module Mixin
           module IncrementalExport
 
+            attr_accessor :previous_page
+
             def self.included(base)
               base.uses :client
             end
@@ -12,6 +14,66 @@ class Sequencer
             def resource_collection
               "::ZendeskAPI::#{resource_klass}".constantize.incremental_export(client, 1)
             end
+
+            def resource_iteration
+              super do |record|
+                # call passed/originally intended block
+                yield(record)
+
+                # add hook to check if object (e.g. ticket) count
+                # update is needed because the request
+                # might have changed
+                update_count
+              end
+            end
+
+            # The source if this is the limitation of not knowing
+            # how much objects there are in total before requesting the endpoint
+            # This is caused by the Zendesk API which only returns max. 1000
+            # per request
+            def update_count
+              update_import_job
+              self.previous_page = next_page
+            end
+
+            def update_import_job
+              return if !update_required?
+
+              state.provide(:import_job, updated_import_job)
+            end
+
+            def klass_key
+              resource_klass.singularize.to_sym
+            end
+
+            def updated_import_job
+
+              import_job.result[klass_key].merge(
+                total: import_job.result[klass_key][:total] + current_request_count
+              )
+              import_job
+            end
+
+            def update_required?
+              return false if previous_page.blank?
+              return false if previous_page == next_page
+
+              current_request_count.present?
+            end
+
+            def current_request_count
+              # access the internal instance method of the
+              # Zendesk collection request to get the current
+              # count of the endpoint (max. 1000)
+              resource_collection_attribute.instance_variable_get(:@count)
+            end
+
+            def next_page
+              # access the internal instance method of the
+              # Zendesk collection request to get the next
+              # page number of the endpoint
+              resource_collection_attribute.instance_variable_get(:@next_page)
+            end
           end
         end
       end

lib/sequencer/unit/import/zendesk/objects_total_count.rb

@@ -17,32 +17,22 @@ class Sequencer
             end
           end
 
-          def request(object)
+          # the special "incremental_export" logic is needed because Zendesk
-            return tickets if object == 'Tickets'
+          # archives records and doesn't return them via e.g. client.tickets
-
-            generic(object)
-          end
-
-          def generic(object)
-            client.send(object.to_s.underscore.to_sym)
-          end
-
-          # this special ticket logic is needed since Zendesk archives tickets
-          # after 120 days and doesn't return them via the client.tickets
           # endpoint as described here:
           # https://github.com/zammad/zammad/issues/558#issuecomment-267951351
-          # the proper way is to use the 'incremental' endpoint which is not available
+          # Counting via the incremental_export endpoint has the limitations
-          # via the ruby gem yet but a pull request is pending:
+          # that it returns max. 1000. That's why we need to update the total
-          # https://github.com/zendesk/zendesk_api_client_rb/pull/287
+          # number while importing in the resource loop
-          # the following workaround is needed to use this functionality
+          def request(object)
-          # Counting Tickets has the limitations that max. 1000 are returned
+            resource_class = "::ZendeskAPI::#{object.singularize}".safe_constantize
-          # that's why we need to update the number when it's exceeded while importing
+            if resource_class.respond_to?(:incremental_export)
-          def tickets
+              # read as: ::ZendeskAPI::Ticket.incremental_export(client, 1)
-            ZendeskAPI::Collection.new(
+              resource_class.incremental_export(client, 1)
-              client,
+            else
-              ZendeskAPI::Ticket,
+              # read as: client.groups
-              path: 'incremental/tickets?start_time=1'
+              client.send(object.to_s.underscore.to_sym)
-            )
+            end
           end
         end
       end

lib/sequencer/unit/import/zendesk/organizations.rb

@@ -4,12 +4,6 @@ class Sequencer
       module Zendesk
         class Organizations < Sequencer::Unit::Import::Zendesk::SubSequence::Object
           include ::Sequencer::Unit::Import::Zendesk::Mixin::IncrementalExport
-
-          private
-
-          def resource_iteration_method
-            :all!
-          end
         end
       end
     end

lib/sequencer/unit/import/zendesk/tickets.rb

@@ -17,62 +17,6 @@ class Sequencer
               ticket_field_map: ticket_field_map,
             )
           end
-
-          def resource_iteration
-            super do |record|
-              # call passed/originally intended block
-              yield(record)
-
-              # add hook to check if ticket count
-              # update is needed because the request
-              # might have changed
-              update_ticket_count
-            end
-          end
-
-          # The source if this is the limitation of not knowing
-          # how much tickets there are in total before requesting the endpoint
-          # This is caused by the Zendesk API which only returns max. 1000
-          # per request
-          def update_ticket_count
-            update_import_job
-            next_page
-          end
-
-          attr_accessor :previous_page
-
-          def update_import_job
-            return if !update_required?
-
-            state.provide(import_job, updated_import_job)
-          end
-
-          def updated_import_job
-            import_job.result[:Tickets].merge(
-              total: import_job.result[:Tickets][:total] + current_request_count
-            )
-          end
-
-          def update_required?
-            return false if previous_page.blank?
-            return false if previous_page == next_page
-
-            current_request_count.present?
-          end
-
-          def current_request_count
-            # access the internal instance method of the
-            # Zendesk collection request to get the current
-            # count of the endpoint (max. 1000)
-            resource_collection_attribute.instance_variable_get(:@count)
-          end
-
-          def next_page
-            # access the internal instance method of the
-            # Zendesk collection request to get the next
-            # page number of the endpoint
-            resource_collection_attribute.instance_variable_get(:@next_page)
-          end
         end
       end
     end