Sutty/trabajo-afectivo
5
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Maintenance: Improve handling of Zendesk API to get the correct count of objects while import.

Browse source
This commit is contained in:
Denny Bresch 2019-08-12 17:07:06 +02:00 committed by Thorsten Eckel
parent fb027c514b
commit 781ffa3209
4 changed files with 76 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

62
lib/sequencer/unit/import/zendesk/mixin/incremental_export.rb
View file

@ -5,6 +5,8 @@ class Sequencer
module Mixin
module IncrementalExport
attr_accessor :previous_page
def self.included(base)
base.uses :client
end
@ -12,6 +14,66 @@ class Sequencer
def resource_collection
"::ZendeskAPI::#{resource_klass}".constantize.incremental_export(client, 1)
end
def resource_iteration
super do |record|
# call passed/originally intended block
yield(record)
# add hook to check if object (e.g. ticket) count
# update is needed because the request
# might have changed
update_count
end
end
# The source if this is the limitation of not knowing
# how much objects there are in total before requesting the endpoint
# This is caused by the Zendesk API which only returns max. 1000
# per request
def update_count
update_import_job
self.previous_page = next_page
end
def update_import_job
return if !update_required?
state.provide(:import_job, updated_import_job)
end
def klass_key
resource_klass.singularize.to_sym
end
def updated_import_job
import_job.result[klass_key].merge(
total: import_job.result[klass_key][:total] + current_request_count
)
import_job
end
def update_required?
return false if previous_page.blank?
return false if previous_page == next_page
current_request_count.present?
end
def current_request_count
# access the internal instance method of the
# Zendesk collection request to get the current
# count of the endpoint (max. 1000)
resource_collection_attribute.instance_variable_get(:@count)
end
def next_page
# access the internal instance method of the
# Zendesk collection request to get the next
# page number of the endpoint
resource_collection_attribute.instance_variable_get(:@next_page)
end
end
end
end

38
lib/sequencer/unit/import/zendesk/objects_total_count.rb
View file

@ -17,32 +17,22 @@ class Sequencer
end
end
def request(object)
return tickets if object == 'Tickets'
generic(object)
end
def generic(object)
client.send(object.to_s.underscore.to_sym)
end
# this special ticket logic is needed since Zendesk archives tickets
# after 120 days and doesn't return them via the client.tickets
# the special "incremental_export" logic is needed because Zendesk
# archives records and doesn't return them via e.g. client.tickets
# endpoint as described here:
# https://github.com/zammad/zammad/issues/558#issuecomment-267951351
# the proper way is to use the 'incremental' endpoint which is not available
# via the ruby gem yet but a pull request is pending:
# https://github.com/zendesk/zendesk_api_client_rb/pull/287
# the following workaround is needed to use this functionality
# Counting Tickets has the limitations that max. 1000 are returned
# that's why we need to update the number when it's exceeded while importing
def tickets
ZendeskAPI::Collection.new(
client,
ZendeskAPI::Ticket,
path: 'incremental/tickets?start_time=1'
)
# Counting via the incremental_export endpoint has the limitations
# that it returns max. 1000. That's why we need to update the total
# number while importing in the resource loop
def request(object)
resource_class = "::ZendeskAPI::#{object.singularize}".safe_constantize
if resource_class.respond_to?(:incremental_export)
# read as: ::ZendeskAPI::Ticket.incremental_export(client, 1)
resource_class.incremental_export(client, 1)
else
# read as: client.groups
client.send(object.to_s.underscore.to_sym)
end
end
end
end

6
lib/sequencer/unit/import/zendesk/organizations.rb
View file

@ -4,12 +4,6 @@ class Sequencer
module Zendesk
class Organizations < Sequencer::Unit::Import::Zendesk::SubSequence::Object
include ::Sequencer::Unit::Import::Zendesk::Mixin::IncrementalExport
private
def resource_iteration_method
:all!
end
end
end
end

56
lib/sequencer/unit/import/zendesk/tickets.rb
View file

@ -17,62 +17,6 @@ class Sequencer
ticket_field_map: ticket_field_map,
)
end
def resource_iteration
super do |record|
# call passed/originally intended block
yield(record)
# add hook to check if ticket count
# update is needed because the request
# might have changed
update_ticket_count
end
end
# The source if this is the limitation of not knowing
# how much tickets there are in total before requesting the endpoint
# This is caused by the Zendesk API which only returns max. 1000
# per request
def update_ticket_count
update_import_job
next_page
end
attr_accessor :previous_page
def update_import_job
return if !update_required?
state.provide(import_job, updated_import_job)
end
def updated_import_job
import_job.result[:Tickets].merge(
total: import_job.result[:Tickets][:total] + current_request_count
)
end
def update_required?
return false if previous_page.blank?
return false if previous_page == next_page
current_request_count.present?
end
def current_request_count
# access the internal instance method of the
# Zendesk collection request to get the current
# count of the endpoint (max. 1000)
resource_collection_attribute.instance_variable_get(:@count)
end
def next_page
# access the internal instance method of the
# Zendesk collection request to get the next
# page number of the endpoint
resource_collection_attribute.instance_variable_get(:@next_page)
end
end
end
end