diff --git a/Gemfile b/Gemfile index de130ccac..449f5ac68 100644 --- a/Gemfile +++ b/Gemfile @@ -80,6 +80,9 @@ gem 'browser' gem 'slack-notifier' gem 'clearbit' gem 'zendesk_api' +gem 'viewpoint' +gem 'rubyntlm', git: 'https://github.com/wimm/rubyntlm.git' +gem 'autodiscover', git: 'https://github.com/thorsteneckel/autodiscover.git' # event machine gem 'eventmachine' diff --git a/Gemfile.lock b/Gemfile.lock index ffba171c8..19b5096f5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,19 @@ +GIT + remote: https://github.com/thorsteneckel/autodiscover.git + revision: 29d713ee0c8c25fcf74c4292ff13fe1fa4d0d827 + specs: + autodiscover (1.0.2) + httpclient + logging + nokogiri + nori + +GIT + remote: https://github.com/wimm/rubyntlm.git + revision: 53969639b87b9e5d5fef560f19cf0d977259591c + specs: + rubyntlm (0.1.2) + GEM remote: https://rubygems.org/ specs: @@ -166,6 +182,7 @@ GEM domain_name (~> 0.5) http-form_data (1.0.3) http_parser.rb (0.6.0) + httpclient (2.8.3) i18n (0.8.6) icalendar (2.4.1) inflection (1.0.0) @@ -181,6 +198,10 @@ GEM rb-fsevent (~> 0.9, >= 0.9.4) rb-inotify (~> 0.9, >= 0.9.7) ruby_dep (~> 1.2) + little-plugger (1.1.4) + logging (2.2.2) + little-plugger (~> 1.1) + multi_json (~> 1.10) loofah (2.0.3) nokogiri (>= 1.5.9) lumberjack (1.0.10) @@ -205,6 +226,7 @@ GEM netrc (0.11.0) nokogiri (1.8.0) mini_portile2 (~> 2.2.0) + nori (2.6.0) notiffany (0.1.1) nenv (~> 0.1) shellany (~> 0.0) @@ -408,6 +430,11 @@ GEM valid_email2 (2.0.0) activemodel (>= 3.2) mail (~> 2.5) + viewpoint (1.1.0) + httpclient + logging + nokogiri + rubyntlm webmock (3.0.1) addressable (>= 2.3.6) crack (>= 0.3.2) @@ -428,6 +455,7 @@ DEPENDENCIES activerecord-nulldb-adapter activerecord-session_store argon2 + autodiscover! autoprefixer-rails biz browser @@ -479,6 +507,7 @@ DEPENDENCIES rb-fsevent rspec-rails rubocop + rubyntlm! sass-rails selenium-webdriver simple-rss @@ -496,6 +525,7 @@ DEPENDENCIES uglifier unicorn valid_email2 + viewpoint webmock writeexcel zendesk_api diff --git a/app/assets/javascripts/app/controllers/_integration/exchange.coffee b/app/assets/javascripts/app/controllers/_integration/exchange.coffee new file mode 100644 index 000000000..26aaef204 --- /dev/null +++ b/app/assets/javascripts/app/controllers/_integration/exchange.coffee @@ -0,0 +1,522 @@ +class Index extends App.ControllerIntegrationBase + featureIntegration: 'exchange_integration' + featureName: 'Exchange' + featureConfig: 'exchange_config' + description: [ + ['This service enables Zammad to connect with your Exchange server.'] + ] + events: + 'change .js-switch input': 'switch' + + render: => + super + new Form( + el: @$('.js-form') + ) + + #new App.ImportJob( + # el: @$('.js-importJob') + # facility: 'exchange' + #) + + new App.HttpLog( + el: @$('.js-log') + facility: 'exchange' + ) + + switch: => + super + active = @$('.js-switch input').prop('checked') + if active + job_start = => + @ajax( + id: 'jobs_config' + type: 'POST' + url: "#{@apiPath}/integration/exchange/job_start" + processData: true + success: (data, status, xhr) => + @render(true) + ) + + App.Delay.set( + job_start, + 600, + 'job_start', + ) + +class Form extends App.Controller + elements: + '.js-lastImport': 'lastImport' + '.js-wizard': 'wizardButton' + events: + 'click .js-wizard': 'startWizard' + 'click .js-start-sync': 'startSync' + + constructor: -> + super + @render() + @lastResult() + @activeDryRun() + + currentConfig: -> + App.Setting.get('exchange_config') || {} + + setConfig: (value) => + App.Setting.set('exchange_config', value, {notify: true}) + @startSync() + + render: (top = false) => + @config = @currentConfig() + + folders = [] + if !_.isEmpty(@config.folders) + for folder_id in @config.folders + folders.push @config.wizardData.backend_folders[folder_id] + + @html App.view('integration/exchange')( + config: @config, + folders: folders + ) + if _.isEmpty(@config) + @$('.js-notConfigured').removeClass('hide') + @$('.js-summary').addClass('hide') + else + @$('.js-notConfigured').addClass('hide') + @$('.js-summary').removeClass('hide') + + if top + a = => + @scrollToIfNeeded($('.content.active .page-header')) + @delay(a, 500) + + startSync: => + @ajax( + id: 'jobs_config' + type: 'POST' + url: "#{@apiPath}/integration/exchange/job_start" + processData: true + success: (data, status, xhr) => + @render(true) + @lastResult() + ) + + startWizard: (e) => + e.preventDefault() + new ConnectionWizard( + container: @el.closest('.content') + config: @config + callback: (config) => + @setConfig(config) + ) + + lastResult: => + @ajax( + id: 'jobs_start_index' + type: 'GET' + url: "#{@apiPath}/integration/exchange/job_start" + processData: true + success: (job, status, xhr) => + if !_.isEmpty(job) + if !@lastResultShowJob || @lastResultShowJob.updated_at != job.updated_at + @lastResultShowJob = job + @lastResultShow(job) + if job.finished_at + @wizardButton.attr('disabled', false) + else + @wizardButton.attr('disabled', true) + @delay(@lastResult, 5000) + ) + + lastResultShow: (job) => + if _.isEmpty(job) + @lastImport.html('') + return + countDone = job.result.created + job.result.updated + job.result.unchanged + job.result.skipped + job.result.failed + if !job.result.roles + job.result.roles = {} + for role_id, statistic of job.result.role_ids + role = App.Role.find(role_id) + job.result.roles[role.displayName()] = statistic + el = $(App.view('integration/exchange_last_import')(job: job, countDone: countDone)) + @lastImport.html(el) + + activeDryRun: => + @ajax( + id: 'jobs_try_index' + type: 'GET' + url: "#{@apiPath}/integration/exchange/job_try" + data: + finished: false + processData: true + success: (job, status, xhr) => + return if _.isEmpty(job) + + # show analyzing + new ConnectionWizard( + container: @el.closest('.content') + config: job.payload + start: 'tryLoop' + callback: (config) => + @wizardButton.attr('disabled', false) + @setConfig(config) + ) + @wizardButton.attr('disabled', true) + ) + +class State + @current: -> + App.Setting.get('exchange_integration') + +class ConnectionWizard extends App.WizardModal + wizardConfig: {} + slideMethod: + 'js-folders': 'foldersShow' + 'js-mapping': 'mappingShow' + + events: + 'submit form.js-discover': 'discover' + 'submit form.js-bind': 'folders' + 'submit form.js-folders': 'mapping' + 'click .js-mapping .js-submitTry': 'mappingChange' + 'click .js-try .js-submitSave': 'save' + 'click .js-close': 'hide' + 'click .js-remove': 'removeRow' + 'click .js-userMappingForm .js-add': 'addUserMapping' + 'click .js-goToSlide': 'goToSlide' + + elements: + '.modal-body': 'body' + '.js-foldersSelect': 'foldersSelect' + '.js-userMappingForm': 'userMappingForm' + '.js-expertForm': 'expertForm' + + constructor: -> + super + + if !_.isEmpty(@config) + @wizardConfig = @config + + if @container + @el.addClass('modal--local') + + @render() + + @el.modal + keyboard: true + show: true + backdrop: true + container: @container + .on + 'show.bs.modal': @onShow + 'shown.bs.modal': @onShown + 'hidden.bs.modal': => + @el.remove() + + if @slide + @showSlide(@slide) + else + @showDiscoverDetails() + + if @start + @[@start]() + + render: => + @html App.view('integration/exchange_wizard')() + + save: (e) => + e.preventDefault() + @callback(@wizardConfig) + @hide(e) + + showSlide: (slide) => + method = @slideMethod[slide] + if method && @[method] + @[method](true) + super + + showDiscoverDetails: => + @$('.js-discover input[name="user"]').val(@wizardConfig.user) + @$('.js-discover input[name="password"]').val(@wizardConfig.password) + + showBindDetails: => + @$('.js-bind input[name="endpoint"]').val(@wizardConfig.endpoint) + @$('.js-bind input[name="user"]').val(@wizardConfig.user) + @$('.js-bind input[name="password"]').val(@wizardConfig.password) + + discover: (e) => + e.preventDefault() + @showSlide('js-connect') + params = @formParam(e.target) + @ajax( + id: 'exchange_discover' + type: 'POST' + url: "#{@apiPath}/integration/exchange/autodiscover" + data: JSON.stringify(params) + processData: true + success: (data, status, xhr) => + if data.result isnt 'ok' + @showSlide('js-discover') + @showAlert('js-discover', data.message) + return + + @wizardConfig.endpoint = data.endpoint + @wizardConfig.user = params.user + @wizardConfig.password = params.password + + @showSlide('js-bind') + @showBindDetails() + + error: (xhr, statusText, error) => + detailsRaw = xhr.responseText + details = {} + if !_.isEmpty(detailsRaw) + details = JSON.parse(detailsRaw) + @showSlide('js-discover') + @showAlert('js-discover', details.error || 'Unable to perform backend.') + ) + + folders: (e) => + e.preventDefault() + @showSlide('js-analyze') + params = @formParam(e.target) + @ajax( + id: 'exchange_folders' + type: 'POST' + url: "#{@apiPath}/integration/exchange/folders" + data: JSON.stringify(params) + processData: true + success: (data, status, xhr) => + if data.result isnt 'ok' + @showSlide('js-bind') + @showAlert('js-bind', data.message) + return + + @wizardConfig.endpoint = params.endpoint + @wizardConfig.user = params.user + @wizardConfig.password = params.password + + # update wizard data + @wizardConfig.wizardData = {} + @wizardConfig.wizardData.backend_folders = data.folders + + @foldersShow() + + error: (xhr, statusText, error) => + detailsRaw = xhr.responseText + details = {} + if !_.isEmpty(detailsRaw) + details = JSON.parse(detailsRaw) + @showSlide('js-bind') + @showAlert('js-bind', details.error || 'Unable to perform backend.') + ) + + foldersShow: (alreadyShown) => + @showSlide('js-folders') if !alreadyShown + @foldersSelect.html(@createColumnSelection('folders', @wizardConfig.wizardData.backend_folders, @wizardConfig.folders)) + + createColumnSelection: (name, options, selected) -> + return App.UiElement.column_select.render( + name: name + null: false + nulloption: false + options: options + value: selected + ) + + mapping: (e) => + e.preventDefault() + @showSlide('js-analyze') + params = @formParam(e.target) + + # folders might be a single selection so we + # have to ensure that is an Array so the + # backend and frontend can handle it properly + if typeof params.folders is 'string' + params.folders = [ params.folders ] + + # add login params + params.endpoint = @wizardConfig.endpoint + params.user = @wizardConfig.user + params.password = @wizardConfig.password + + @ajax( + id: 'exchange_mapping' + type: 'POST' + url: "#{@apiPath}/integration/exchange/mapping" + data: JSON.stringify(params) + processData: true + success: (data, status, xhr) => + if data.result isnt 'ok' + @showSlide('js-folders') + @showAlert('js-folders', data.message) + return + + attributes = {} + for key, value of App.User.attributesGet() + continue if key == 'login' + if (value.tag is 'input' || value.tag is 'richtext' || value.tag is 'textarea') && value.type isnt 'password' + attributes[key] = value.display || key + + @wizardConfig.wizardData.attributes = attributes + @wizardConfig.folders = params.folders + @wizardConfig.wizardData.backend_attributes = data.attributes + + @mappingShow() + + error: (xhr, statusText, error) => + detailsRaw = xhr.responseText + details = {} + if !_.isEmpty(detailsRaw) + details = JSON.parse(detailsRaw) + @showSlide('js-folders') + @showAlert('js-folders', details.error || 'Unable to perform backend.') + ) + + mappingShow: (alreadyShown) => + @showSlide('js-mapping') if !alreadyShown + user_attribute_map = @wizardConfig.attributes + + if _.isEmpty(user_attribute_map) + user_attribute_map = + given_name: 'firstname' + surname: 'lastname' + 'email_addresses.emailaddress1': 'email' + 'phone_numbers.businessphone': 'phone' + + @userMappingForm.find('tbody tr.js-entry').remove() + @userMappingForm.find('tbody tr').before(@buildRowsUserMap(user_attribute_map)) + + mappingChange: (e) => + e.preventDefault() + + # user map + attributes = @formParam(@userMappingForm) + for key in ['source', 'dest'] + if !_.isArray(attributes[key]) + attributes[key] = [attributes[key]] + attributes_local = + item_id: 'login' + length = attributes.source.length-1 + for count in [0..length] + if attributes.source[count] && attributes.dest[count] + attributes_local[attributes.source[count]] = attributes.dest[count] + @wizardConfig.attributes = attributes_local + + @tryShow() + + buildRowsUserMap: (user_attribute_map) => + + # show static login row + userUidDisplayValue = @wizardConfig.wizardData.backend_attributes['item_id'] + el = [ + $(App.view('integration/ldap_user_attribute_row_read_only')( + key: userUidDisplayValue, + value: 'Login' + )) + ] + + for source, dest of user_attribute_map + continue if source == 'item_id' + continue if !(source of @wizardConfig.wizardData.backend_attributes) + el.push @buildRowUserAttribute(source, dest) + el + + buildRowUserAttribute: (source, dest) => + el = $(App.view('integration/exchange_user_attribute_row')()) + el.find('.js-exchangeAttribute').html(@createSelection('source', @wizardConfig.wizardData.backend_attributes, source)) + el.find('.js-userAttribute').html(@createSelection('dest', @wizardConfig.wizardData.attributes, dest)) + el + + createSelection: (name, options, selected, unknown) -> + return App.UiElement.searchable_select.render( + name: name + multiple: false + limit: 100 + null: false + nulloption: false + options: options + value: selected + unknown: unknown + class: 'form-control--small' + ) + + removeRow: (e) -> + e.preventDefault() + $(e.target).closest('tr').remove() + + addUserMapping: (e) => + e.preventDefault() + @userMappingForm.find('tbody tr').last().before(@buildRowUserAttribute()) + + tryShow: (e) => + if e + e.preventDefault() + @showSlide('js-analyze') + + # create import job + @ajax( + id: 'exchange_try' + type: 'POST' + url: "#{@apiPath}/integration/exchange/job_try" + data: JSON.stringify(@wizardConfig) + processData: true + success: (data, status, xhr) => + @tryLoop() + ) + + tryLoop: => + @showSlide('js-dry') + @ajax( + id: 'jobs_try_index' + type: 'GET' + url: "#{@apiPath}/integration/exchange/job_try" + data: + finished: true + processData: true + success: (job, status, xhr) => + if job.result && (job.result.error || job.result.info) + @showSlide('js-error') + @showAlert('js-error', (job.result.error || job.result.info)) + return + + total = 0 + if job.result && _.keys(job.result).length > 0 + @$('.js-preprogress').addClass('hide') + @$('.js-analyzing').removeClass('hide') + + analized = 0 + total = job.result.sum + for action, sum of job.result + continue if action == 'folders' + continue if action == 'sum' + analized += sum + + @$('.js-progress progress').attr('value', analized) + @$('.js-progress progress').attr('max', total) + + if job.finished_at + # reset initial state in case the back button is used + @$('.js-preprogress').removeClass('hide') + @$('.js-analyzing').addClass('hide') + + @tryResult(job, total) + else + @delay(@tryLoop, 4000) + ) + + tryResult: (job, total) => + @showSlide('js-try') + el = $(App.view('integration/exchange_summary')(job: job, countDone: total)) + @el.find('.js-summary').html(el) + +App.Config.set( + 'IntegrationExchange' + { + name: 'Exchange' + target: '#system/integration/exchange' + description: 'Exchange integration for contacts management.' + controller: Index + state: State + } + 'NavBarIntegrations' +) diff --git a/app/assets/javascripts/app/controllers/_integration/ldap.coffee b/app/assets/javascripts/app/controllers/_integration/ldap.coffee index dc0bbbdef..83a61dc98 100644 --- a/app/assets/javascripts/app/controllers/_integration/ldap.coffee +++ b/app/assets/javascripts/app/controllers/_integration/ldap.coffee @@ -28,13 +28,20 @@ class Index extends App.ControllerIntegrationBase super active = @$('.js-switch input').prop('checked') if active - @ajax( - id: 'jobs_config' - type: 'POST' - url: "#{@apiPath}/integration/ldap/job_start" - processData: true - success: (data, status, xhr) => - @render(true) + job_start = => + @ajax( + id: 'jobs_config' + type: 'POST' + url: "#{@apiPath}/integration/ldap/job_start" + processData: true + success: (data, status, xhr) => + @render(true) + ) + + App.Delay.set( + job_start, + 600, + 'job_start', ) class Form extends App.Controller @@ -91,6 +98,7 @@ class Form extends App.Controller processData: true success: (data, status, xhr) => @render(true) + @lastResult() ) startWizard: (e) => diff --git a/app/assets/javascripts/app/views/integration/exchange.jst.eco b/app/assets/javascripts/app/views/integration/exchange.jst.eco new file mode 100644 index 000000000..69385e1cf --- /dev/null +++ b/app/assets/javascripts/app/views/integration/exchange.jst.eco @@ -0,0 +1,71 @@ +
+
+

<%- @T('No %s configured.', 'Exchange') %>

+ +
+
+

<%- @T('Settings') %>

+ + + + + + + + + + +
<%- @T('Name') %> + <%- @T('Value') %> +
<%- @T('Endpoint') %> + <%= @config.endpoint %> +
<%- @T('User') %> + <%= @config.user %> +
<%- @T('Password') %> + <%= @M(@config.password) %> +
+ +

<%- @T('Mapping') %>

+ +

<%- @T('Folders') %>

+ <% if _.isEmpty(@folders): %> + +
<%- @T('No Entries') %> +
+ <% else: %> + + + + + + +
<%- @T('Folder') %> + <% for folder_name in @folders: %> +
<%= folder_name %> + <% end %> +
+ <% end %> + +

<%- @T('User') %>

+ <% if _.isEmpty(@config.attributes): %> + +
<%- @T('No Entries') %> +
+ <% else: %> + + + + + + +
<%- @T('Exchange') %> + <%- @T('Zammad') %> + <% for key, value of @config.attributes: %> +
<%= key %> + <%= value %> + <% end %> +
+ <% end %> + + +
diff --git a/app/assets/javascripts/app/views/integration/exchange_last_import.jst.eco b/app/assets/javascripts/app/views/integration/exchange_last_import.jst.eco new file mode 100644 index 000000000..8323972ad --- /dev/null +++ b/app/assets/javascripts/app/views/integration/exchange_last_import.jst.eco @@ -0,0 +1,54 @@ +
+

<%- @T('Last sync') %>

+ <% if _.isEmpty(@job.started_at): %> + <% if @job.result && @job.result.error: %> + + <% else if @job.result && @job.result.info: %> + + <% else: %> +

<%- @T('Job is waiting to get started...') %>

+ <% end %> + <% else: %> + <% if @job.finished_at: %> +

<%- @Ttimestamp(@job.started_at) %> - <%- @Ttimestamp(@job.finished_at) %>

+ <% if @job.result && @job.result.error: %> + + <% else if @job.result && @job.result.info: %> + + <% end %> + <% else: %> + <% if @job.result && @job.result.error: %> +

<%- @Ttimestamp(@job.started_at) %>

+ + <% else if !@countDone: %> +

<%- @Ttimestamp(@job.started_at) %> - <%- @T('Counting entries. This may take a while.') %>

+ <% else: %> +

<%- @Ttimestamp(@job.started_at) %> - <%- @T('Running...') %>

+
+ +
+ <% end %> + <% end %> + <% if !_.isEmpty(@job.result) && @countDone: %> + + <% end %> + <% if @job.finished_at: %> + + <% end %> + <% end %> +
diff --git a/app/assets/javascripts/app/views/integration/exchange_summary.jst.eco b/app/assets/javascripts/app/views/integration/exchange_summary.jst.eco new file mode 100644 index 000000000..0abf47bdf --- /dev/null +++ b/app/assets/javascripts/app/views/integration/exchange_summary.jst.eco @@ -0,0 +1,16 @@ + diff --git a/app/assets/javascripts/app/views/integration/exchange_user_attribute_row.jst.eco b/app/assets/javascripts/app/views/integration/exchange_user_attribute_row.jst.eco new file mode 100644 index 000000000..1d9dd1826 --- /dev/null +++ b/app/assets/javascripts/app/views/integration/exchange_user_attribute_row.jst.eco @@ -0,0 +1,7 @@ + + + + +
+ <%- @Icon('trash') %> <%- @T('Remove') %> +
diff --git a/app/assets/javascripts/app/views/integration/exchange_wizard.jst.eco b/app/assets/javascripts/app/views/integration/exchange_wizard.jst.eco new file mode 100644 index 000000000..25cdbcade --- /dev/null +++ b/app/assets/javascripts/app/views/integration/exchange_wizard.jst.eco @@ -0,0 +1,258 @@ +
+ +
+
+
+ <%- @Icon('diagonal-cross') %> +
+

<%- @T('Exchange') %> <%- @T('Configuration') %>

+
+
+
+ + + + + + + + + +
<%- @T('Name') %> + <%- @T('Value') %> +
<%- @T('User') %> + +
<%- @T('Password') %> + +
+
+
+
+
+ +
+
+
+ +
+
+
+ <%- @Icon('diagonal-cross') %> +
+

<%- @T('Exchange') %> <%- @T('Configuration') %>

+
+
+
+

+ <%- @T('Connecting ...') %> +

+
+
+
+
+ +
+
+
+ <%- @Icon('diagonal-cross') %> +
+

<%- @T('Exchange') %> <%- @T('Configuration') %>

+
+
+
+ + + + + + + + + + +
<%- @T('Name') %> + <%- @T('Value') %> +
<%- @T('Endpoint') %> + + +
<%- @T('User') %> + +
<%- @T('Password') %> + +
+
+
+
+
+ +
+
+
+ +
+
+
+ <%- @Icon('diagonal-cross') %> +
+

<%- @T('Exchange') %> <%- @T('Folders') %>

+
+
+
+ + +
+
+ +
+
+
+
+ +
+
+
+
+ <%- @T('Go Back') %> +
+
+ +
+
+
+ +
+
+
+ <%- @Icon('diagonal-cross') %> +
+

<%- @T('Exchange') %> <%- @T('Configuration') %>

+
+
+
+

+ <%- @T('Analyzing structure...') %> +

+
+
+
+
+ +
+
+
+ <%- @Icon('diagonal-cross') %> +
+

<%- @T('Exchange') %> <%- @T('Mapping') %>

+
+
+
+ + +

<%- @T('User') %>

+
+ + + + + + + + + + + + +
<%- @T('%s Attribute', 'Exchange') %> + <%- @T('%s Attribute', 'Zammad') %> + <%- @T('Action') %> +
+
+ <%- @Icon('plus-small') %> <%- @T('Add') %> +
+
+
+ +
+
+
+
+ <%- @T('Go Back') %> +
+
+ +
+
+
+ +
+
+
+ <%- @Icon('diagonal-cross') %> +
+

<%- @T('Exchange') %> <%- @T('Configuration') %>

+
+
+
+
+

+ + <%- @T('Counting entries. This may take a while.') %> +

+
+
+

+ <%- @T('Analyzing entries with given configuration...') %> +

+
+ +
+
+
+
+
+
+ +
+
+
+ <%- @Icon('diagonal-cross') %> +
+

<%- @T('Exchange') %> <%- @T('Configuration') %>

+
+
+
+ +

<%- @T('With your current configuration the following will happen') %>:

+
+
+
+
+
+ <%- @T('Go Back') %> +
+
+ +
+
+
+ +
+
+
+ <%- @Icon('diagonal-cross') %> +
+

<%- @T('Exchange') %>

+
+
+
+ +
+
+
+
+ +
+
+
+ +
diff --git a/app/controllers/concerns/integration/import_job_base.rb b/app/controllers/concerns/integration/import_job_base.rb new file mode 100644 index 000000000..493351ec7 --- /dev/null +++ b/app/controllers/concerns/integration/import_job_base.rb @@ -0,0 +1,89 @@ +module Integration::ImportJobBase + extend ActiveSupport::Concern + + def job_try_index + job_index( + dry_run: true, + take_finished: params[:finished] == 'true' + ) + end + + def job_try_create + ImportJob.dry_run(name: import_backend_namespace, payload: payload_dry_run) + render json: { + result: 'ok', + } + end + + def job_start_index + job_index(dry_run: false) + end + + def job_start_create + if !ImportJob.exists?(name: import_backend_namespace, finished_at: nil) + job = ImportJob.create(name: import_backend_namespace, payload: payload_import) + job.delay.start + end + render json: { + result: 'ok', + } + end + + def payload_dry_run + params + end + + def payload_import + import_setting + end + + private + + def answer_with + result = yield + render json: result.merge(result: 'ok') + rescue => e + logger.error(e) + render json: { + result: 'failed', + message: e.message, + } + end + + def import_setting + Setting.get(import_setting_name) + end + + def import_setting_name + "#{import_backend_name.downcase}_config" + end + + def import_backend_namespace + "Import::#{import_backend_name}" + end + + def import_backend_name + self.class.name.split('::').last.sub('Controller', '') + end + + def job_index(dry_run:, take_finished: true) + job = ImportJob.find_by( + name: import_backend_namespace, + dry_run: dry_run, + finished_at: nil + ) + if !job && take_finished + job = ImportJob.where( + name: import_backend_namespace, + dry_run: dry_run + ).order(created_at: :desc).limit(1).first + end + + if job + model_show_render_item(job) + else + render json: {} + end + end + +end diff --git a/app/controllers/integration/exchange_controller.rb b/app/controllers/integration/exchange_controller.rb new file mode 100644 index 000000000..66e4ca30d --- /dev/null +++ b/app/controllers/integration/exchange_controller.rb @@ -0,0 +1,66 @@ +# Copyright (C) 2012-2016 Zammad Foundation, http://zammad-foundation.org/ + +class Integration::ExchangeController < ApplicationController + include Integration::ImportJobBase + + prepend_before_action { authentication_check(permission: 'admin.integration.exchange') } + + def autodiscover + answer_with do + client = Autodiscover::Client.new( + email: params[:user], + password: params[:password], + ) + + { + endpoint: client.autodiscover.ews_url, + } + end + end + + def folders + answer_with do + Sequencer.process('Import::Exchange::AvailableFolders', + parameters: { + ews_config: { + endpoint: params[:endpoint], + user: params[:user], + password: params[:password], + } + }) + end + end + + def mapping + answer_with do + Sequencer.process('Import::Exchange::AttributesExamples', + parameters: { + ews_folder_ids: params[:folders], + ews_config: { + endpoint: params[:endpoint], + user: params[:user], + password: params[:password], + } + }) + end + end + + private + + # currently a workaround till LDAP is migrated to Sequencer + def payload_dry_run + { + ews_attributes: params[:attributes], + ews_folder_ids: params[:folders], + ews_config: { + endpoint: params[:endpoint], + user: params[:user], + password: params[:password], + } + } + end + + def payload_import + nil + end +end diff --git a/app/controllers/integration/ldap_controller.rb b/app/controllers/integration/ldap_controller.rb index 16d2393fb..cbf30654c 100644 --- a/app/controllers/integration/ldap_controller.rb +++ b/app/controllers/integration/ldap_controller.rb @@ -4,6 +4,8 @@ require 'ldap/user' require 'ldap/group' class Integration::LdapController < ApplicationController + include Integration::ImportJobBase + prepend_before_action { authentication_check(permission: 'admin.integration.ldap') } def discover @@ -60,48 +62,4 @@ class Integration::LdapController < ApplicationController message: e.message, } end - - def job_try_index - job_index( - dry_run: true, - take_finished: params[:finished] == 'true' - ) - end - - def job_try_create - ImportJob.dry_run(name: 'Import::Ldap', payload: params) - render json: { - result: 'ok', - } - end - - def job_start_index - job_index(dry_run: false) - end - - def job_start_create - backend = 'Import::Ldap' - if !ImportJob.exists?(name: backend, finished_at: nil) - job = ImportJob.create(name: backend, payload: Setting.get('ldap_config')) - job.delay.start - end - render json: { - result: 'ok', - } - end - - private - - def job_index(dry_run:, take_finished: true) - job = ImportJob.find_by(name: 'Import::Ldap', dry_run: dry_run, finished_at: nil) - if !job && take_finished - job = ImportJob.where(name: 'Import::Ldap', dry_run: dry_run).order(created_at: :desc).limit(1).first - end - - if job - model_show_render_item(job) - else - render json: {} - end - end end diff --git a/config/routes/integration_exchange.rb b/config/routes/integration_exchange.rb new file mode 100644 index 000000000..0c84a90f8 --- /dev/null +++ b/config/routes/integration_exchange.rb @@ -0,0 +1,11 @@ +Zammad::Application.routes.draw do + api_path = Rails.configuration.api_path + + match api_path + '/integration/exchange/autodiscover', to: 'integration/exchange#autodiscover', via: :post + match api_path + '/integration/exchange/folders', to: 'integration/exchange#folders', via: :post + match api_path + '/integration/exchange/mapping', to: 'integration/exchange#mapping', via: :post + match api_path + '/integration/exchange/job_try', to: 'integration/exchange#job_try_index', via: :get + match api_path + '/integration/exchange/job_try', to: 'integration/exchange#job_try_create', via: :post + match api_path + '/integration/exchange/job_start', to: 'integration/exchange#job_start_index', via: :get + match api_path + '/integration/exchange/job_start', to: 'integration/exchange#job_start_create', via: :post +end diff --git a/db/migrate/20170629000001_exchange_integration.rb b/db/migrate/20170629000001_exchange_integration.rb new file mode 100644 index 000000000..88447f2c6 --- /dev/null +++ b/db/migrate/20170629000001_exchange_integration.rb @@ -0,0 +1,51 @@ +class ExchangeIntegration < ActiveRecord::Migration + def up + + # return if it's a new setup + return if !Setting.find_by(name: 'system_init_done') + + Setting.set('import_backends', ['Import::Ldap', 'Import::Exchange']) + + Setting.create_if_not_exists( + title: 'Exchange config', + name: 'exchange_config', + area: 'Integration::Exchange', + description: 'Defines the Exchange config.', + options: {}, + state: {}, + preferences: { + prio: 2, + permission: ['admin.integration'], + }, + frontend: false, + ) + Setting.create_if_not_exists( + title: 'Exchange integration', + name: 'exchange_integration', + area: 'Integration::Switch', + description: 'Defines if Exchange is enabled or not.', + options: { + form: [ + { + display: '', + null: true, + name: 'exchange_integration', + tag: 'boolean', + options: { + true => 'yes', + false => 'no', + }, + }, + ], + }, + state: false, + preferences: { + prio: 1, + authentication: true, + permission: ['admin.integration'], + }, + frontend: true + ) + end + +end diff --git a/db/seeds/settings.rb b/db/seeds/settings.rb index 9181600fa..628516ab6 100644 --- a/db/seeds/settings.rb +++ b/db/seeds/settings.rb @@ -2401,7 +2401,7 @@ Setting.create_if_not_exists( area: 'Import', description: 'A list of active import backends that get scheduled automatically.', options: {}, - state: ['Import::Ldap'], + state: ['Import::Ldap', 'Import::Exchange'], preferences: { permission: ['admin'], }, @@ -2874,6 +2874,46 @@ Setting.create_if_not_exists( }, frontend: true ) +Setting.create_if_not_exists( + title: 'Exchange config', + name: 'exchange_config', + area: 'Integration::Exchange', + description: 'Defines the Exchange config.', + options: {}, + state: {}, + preferences: { + prio: 2, + permission: ['admin.integration'], + }, + frontend: false, +) +Setting.create_if_not_exists( + title: 'Exchange integration', + name: 'exchange_integration', + area: 'Integration::Switch', + description: 'Defines if Exchange is enabled or not.', + options: { + form: [ + { + display: '', + null: true, + name: 'exchange_integration', + tag: 'boolean', + options: { + true => 'yes', + false => 'no', + }, + }, + ], + }, + state: false, + preferences: { + prio: 1, + authentication: true, + permission: ['admin.integration'], + }, + frontend: true +) Setting.create_if_not_exists( title: 'LDAP config', name: 'ldap_config', diff --git a/lib/import/exchange.rb b/lib/import/exchange.rb new file mode 100644 index 000000000..cd8d307cb --- /dev/null +++ b/lib/import/exchange.rb @@ -0,0 +1,13 @@ +# Copyright (C) 2012-2016 Zammad Foundation, http://zammad-foundation.org/ + +module Import + class Exchange < Import::IntegrationBase + include Import::Mixin::Sequence + + private + + def sequence_name + 'Import::Exchange::FolderContacts' + end + end +end diff --git a/lib/import/exchange/folder.rb b/lib/import/exchange/folder.rb new file mode 100644 index 000000000..7a4559f0a --- /dev/null +++ b/lib/import/exchange/folder.rb @@ -0,0 +1,76 @@ +require 'mixin/rails_logger' + +module Import + class Exchange + class Folder + include ::Mixin::RailsLogger + + def initialize(connection) + @connection = connection + @lookup_map = {} + end + + def id_folder_map + @id_folder_map ||= all.collect do |folder| + [folder.id, folder] + end.to_h + + # duplicate object to avoid errors where keys get + # added via #get_folder while iterating over + # the result of this method + @lookup_map = @id_folder_map.dup + @id_folder_map + end + + def find(id) + @lookup_map[id] ||= @connection.get_folder(id) + end + + def all + # request folders only if neccessary and store the result + @all ||= children(%i(root msgfolderroot publicfoldersroot)) + end + + def children(parent_identifiers) + parent_identifiers.each_with_object([]) do |parent_identifier, result| + + child_folders = request_children(parent_identifier) + + next if child_folders.blank? + + child_folder_ids = child_folders.collect(&:id) + child_folders += children(child_folder_ids) + + result.concat(child_folders) + end + end + + def display_path(folder) + display_name = folder.display_name + return display_name if !folder.parent_folder_id + + parent_folder = find(folder.parent_folder_id) + return display_name if !parent_folder + + parent_folder = id_folder_map[folder.parent_folder_id] + return display_name if !parent_folder + + # recursive + parent_folder_path = display_path(parent_folder) + + "#{parent_folder_path} -> #{display_name}" + rescue Viewpoint::EWS::EwsError + folder.display_name + end + + private + + def request_children(parent_identifier) + @connection.folders(root: parent_identifier) + rescue Viewpoint::EWS::EwsFolderNotFound => e + logger.warn(e) + nil + end + end + end +end diff --git a/lib/import/exchange/item_attributes.rb b/lib/import/exchange/item_attributes.rb new file mode 100644 index 000000000..bc0d19ec0 --- /dev/null +++ b/lib/import/exchange/item_attributes.rb @@ -0,0 +1,111 @@ +module Import + class Exchange + class ItemAttributes + + def self.extract(resource) + new(resource).extract + end + + def initialize(resource) + @resource = resource + end + + def extract + @attributes ||= begin + properties = @resource.get_all_properties! + result = normalize(properties) + flattened = flatten(result) + booleanized = booleanize_values(flattened) + end + end + + private + + def booleanize_values(properties) + properties.each do |key, value| + if value.is_a?(String) + next if !%w(true false).include?(value) + properties[key] = value == 'true' + elsif value.is_a?(Hash) + properties[key] = booleanize_values(value) + end + end + end + + def normalize(properties) + result = {} + properties.each do |key, value| + + next if key == :body + + if value[:text] + result[key] = value[:text] + elsif value[:attribs] + result[key] = value[:attribs] + elsif value[:elems] + result[key] = sub_elems(value[:elems]) + end + end + + result + end + + def sub_elems(elems) + result = {} + elems.each do |elem| + if elem[:entry] + result.merge!( sub_elem_entry( elem[:entry] ) ) + else + result.merge!( normalize(elem) ) + end + end + result + end + + def sub_elem_entry(entry) + entry_value = {} + if entry[:elems] + entry_value = sub_elems(entry[:elems]) + end + + if entry[:text] + entry_value[:text] = entry[:text] + end + + if entry[:attribs].present? + entry_value.merge!(entry[:attribs]) + end + + entry_key = entry_value.delete(:key) + { + entry_key => entry_value + } + end + + def flatten(properties, prefix: nil) + + result = {} + properties.each do |key, value| + + result_key = key + if prefix + result_key = if %i(text id).include?(key) && ( !result[result_key] || result[result_key] == value ) + prefix + else + "#{prefix}.#{key}".to_sym + end + end + result_key = result_key.to_s.downcase + + if value.is_a?(Hash) + sub_result = flatten(value, prefix: result_key) + result.merge!(sub_result) + else + result[result_key] = value.to_s + end + end + result + end + end + end +end diff --git a/lib/import/helper/attributes_examples.rb b/lib/import/helper/attributes_examples.rb new file mode 100644 index 000000000..1ae3c00a9 --- /dev/null +++ b/lib/import/helper/attributes_examples.rb @@ -0,0 +1,70 @@ +module Import + module Helper + class AttributesExamples + attr_reader :examples, :enough, :max_unkown + + def initialize(&block) + @max_unkown = 50 + @no_new_counter = 1 + @examples = {} + @known = [] + + # Support both builder styles: + # + # Import::Helper::AttributesExamples.new do + # extract(attributes) + # end + # + # and + # + # Import::Helper::AttributesExamples.new do |extractor| + # extractor.extract(attributes) + # end + return if !block_given? + if block.arity.zero? + instance_eval(&block) + else + yield self + end + end + + def extract(attributes) + unknown = attributes.keys - @known + + return if !unknown?(unknown) + + store(attributes, unknown) + + @known.concat(unknown) + @no_new_counter = 0 + end + + private + + def unknown?(unknown) + return true if unknown.present? + + @no_new_counter += 1 + + # check max 50 entries with no or no new attributes in a row + @enough_examples = @no_new_counter != 50 + + false + end + + def store(attributes, unknown) + unknown.each do |attribute| + value = attributes[attribute] + + next if value.nil? + + example = value.to_s.force_encoding('UTF-8').encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '?') + example.gsub!(/^(.{20,}?).*$/m, '\1...') + + @examples[attribute] = "#{attribute} (e. g. #{example})" + end + end + + end + end +end diff --git a/lib/import/integration_base.rb b/lib/import/integration_base.rb new file mode 100644 index 000000000..ac995f9dd --- /dev/null +++ b/lib/import/integration_base.rb @@ -0,0 +1,143 @@ +# Copyright (C) 2012-2016 Zammad Foundation, http://zammad-foundation.org/ + +module Import + + # This base class handles regular integrations. + # It provides generic interfaces for settings and active state. + # It ensures that all requirements for a regular integration are met before a import can start. + # It handles the case of an Scheduler interruption. + # + # It's required to implement the +start_import+ method which only has to start the import. + class IntegrationBase < Import::Base + + def self.inherited(subclass) + subclass.extend(Forwardable) + + # delegate instance methods to the generic class implementations + subclass.delegate [:identifier, :active?, :config, :display_name] => subclass + end + + # Defines the integration identifier used for + # automatic config lookup and error message generation. + # + # @example + # Import::Ldap.identifier + # #=> "Ldap" + # + # return [String] + def self.identifier + name.split('::').last + end + + # Provides the name that is used in texts visible to the user. + # + # @example + # Import::Exchange.display_name + # #=> "Exchange" + # + # return [String] + def self.display_name + identifier + end + + # Checks if the integration is active. + # + # @example + # Import::Ldap.active? + # #=> true + # + # return [Boolean] + def self.active? + Setting.get("#{identifier.downcase}_integration") || false + end + + # Provides the integration configuration. + # + # @example + # Import::Ldap.config + # #=> {"ssl_verify"=>true, "host_url"=>"ldaps://192...", ...} + # + # return [Hash] the configuration + def self.config + Setting.get("#{identifier.downcase}_config") || {} + end + + # Stores the integration configuration. + # + # @example + # Import::Ldap.config = {"ssl_verify"=>true, "host_url"=>"ldaps://192...", ...} + # + # return [nil] + def self.config=(value) + Setting.set("#{identifier.downcase}_config", value) + end + + # Checks if the integration is activated and configured. + # Otherwise it won't get queued since it will display + # an error which is confusing and wrong. + # + # @example + # Import::Ldap.queueable? + # #=> true + # + # return [Boolean] + def self.queueable? + active? && config.present? + end + + # Starts a live or dry run import. + # + # @example + # instance = Import::Ldap.new(import_job) + # + # @raise [RuntimeError] Raised if an import should start but the integration is disabled + # + # return [nil] + def start + return if !requirements_completed? + start_import + end + + # Gets called when the Scheduler gets (re-)started and an ImportJob was still + # in the queue. The job will always get restarted to avoid the gap till the next + # run triggered by the Scheduler. The result will get updated to inform the user + # in the agent interface result view. + # + # @example + # instance = Import::Ldap.new(import_job) + # instance.reschedule?(delayed_job) + # #=> true + # + # return [true] + def reschedule?(_delayed_job) + inform('Restarting due to scheduler restart.') + true + end + + private + + def start_import + raise "Missing implementation of method '#{__method__}' for #{self.class.name}" + end + + def requirements_completed? + return true if @import_job.dry_run + + if !active? + message = "Sync cancelled. #{display_name} integration deactivated. Activate via the switch." + elsif config.blank? && @import_job.payload.blank? + message = "Sync cancelled. #{display_name} configration or ImportJob payload missing." + end + + return true if !message + inform(message) + false + end + + def inform(message) + @import_job.update_attribute(:result, { + info: message + }) + end + end +end diff --git a/lib/import/ldap.rb b/lib/import/ldap.rb index 5b82e60a4..882981bb6 100644 --- a/lib/import/ldap.rb +++ b/lib/import/ldap.rb @@ -4,50 +4,17 @@ require 'ldap' require 'ldap/group' module Import - class Ldap < Import::Base + class Ldap < Import::IntegrationBase - # Checks if the integration is activated and configured. - # Otherwise it won't get queued since it will display - # an error which is confusing and wrong. + # Provides the name that is used in texts visible to the user. # # @example - # Import::LDAP.queueable? - # #=> true + # Import::Ldap.display_name + # #=> "LDAP" # - # return [Boolean] - def self.queueable? - Setting.get('ldap_integration') && Setting.get('ldap_config').present? - end - - # Starts a live or dry run LDAP import. - # - # @example - # instance = Import::LDAP.new(import_job) - # - # @raise [RuntimeError] Raised if an import should start but the ldap integration is disabled - # - # return [nil] - def start - return if !requirements_completed? - start_import - end - - # Gets called when the Scheduler gets (re-)started and a LDAP ImportJob was still - # in the queue. The job will always get restarted to avoid the gap till the next - # run triggered by the Scheduler. The result will get updated to inform the user - # in the agent interface result view. - # - # @example - # instance = Import::LDAP.new(import_job) - # instance.reschedule?(delayed_job) - # #=> true - # - # return [true] - def reschedule?(_delayed_job) - @import_job.update_attribute(:result, { - info: 'Restarting due to scheduler restart.' - }) - true + # return [String] + def self.display_name + identifier.upcase end private @@ -63,23 +30,5 @@ module Import @import_job.result = Import::Ldap::UserFactory.statistics end - - def requirements_completed? - return true if @import_job.dry_run - - if !Setting.get('ldap_integration') - message = 'Sync cancelled. LDAP integration deactivated. Activate via the switch.' - elsif Setting.get('ldap_config').blank? && @import_job.payload.blank? - message = 'Sync cancelled. LDAP configration or ImportJob payload missing.' - end - - return true if !message - - @import_job.update_attribute(:result, { - info: message - }) - - false - end end end diff --git a/lib/import/mixin/sequence.rb b/lib/import/mixin/sequence.rb new file mode 100644 index 000000000..262933e44 --- /dev/null +++ b/lib/import/mixin/sequence.rb @@ -0,0 +1,19 @@ +module Import + module Mixin + module Sequence + private + + def sequence_name + raise "Missing implementation of '#{__method__}' method for '#{self.class.name}'" + end + + def process + Sequencer.process(sequence_name, + parameters: { + import_job: @import_job + }) + end + alias start_import process + end + end +end diff --git a/lib/mixin/instance_wrapper.rb b/lib/mixin/instance_wrapper.rb new file mode 100644 index 000000000..96e8ff8e3 --- /dev/null +++ b/lib/mixin/instance_wrapper.rb @@ -0,0 +1,43 @@ +module Mixin + # This modules enables to redirect all calls to methods that are + # not defined to the declared instance variable. This comes handy + # when you wan't extend a Ruby core class like Hash. + # To inherit directly from such classes is a bad idea and should be avoided. + # This way allows it indirectly. + module InstanceWrapper + module ClassMethods + # Creates the class macro `wrap` that activates + # the wrapping for the given instance variable name. + # + # @param [Symbol] variable the name of the instance variable to wrap around + # + # @example + # wrap :@some_hash + # + # @return [nil] + def wrap(variable) + define_method(:instance) { + instance_variable_get(variable) + } + end + end + + def self.included(base) + base.extend(ClassMethods) + end + + private + + def method_missing(method, *args, &block) + if instance.respond_to?(method) + instance.send(method, *args, &block) + else + super + end + end + + def respond_to_missing?(method_sym, include_all) + instance.respond_to?(method_sym, include_all) + end + end +end diff --git a/lib/mixin/rails_logger.rb b/lib/mixin/rails_logger.rb new file mode 100644 index 000000000..e014f1d08 --- /dev/null +++ b/lib/mixin/rails_logger.rb @@ -0,0 +1,9 @@ +module Mixin + module RailsLogger + extend Forwardable + extend SingleForwardable + + instance_delegate [:logger] => self + single_delegate [:logger] => :Rails + end +end diff --git a/lib/mixin/required_sub_paths.rb b/lib/mixin/required_sub_paths.rb new file mode 100644 index 000000000..fa436749e --- /dev/null +++ b/lib/mixin/required_sub_paths.rb @@ -0,0 +1,44 @@ +module Mixin + module RequiredSubPaths + + def self.included(_base) + path = caller_locations.first.path + sub_path = File.join(File.dirname(path), File.basename(path, '.rb')) + eager_load_recursive(sub_path) + end + + # Loads a directory recursivly. + # The specialty of this method is that it will first load all + # files in a directory and then start with the sub directories. + # This is needed since otherwise some parent namespaces might not + # be initialized yet. + # + # The cause of this is that Rails autoload doesn't work properly + # for same named classes or modules in different namespaces. + # Here is a good description how autoload works: + # http://urbanautomaton.com/blog/2013/08/27/rails-autoloading-hell/ + # + # This avoids a) Rails autoloading issues and b) require '...' workarounds + def self.eager_load_recursive(path) + + excluded = ['.', '..'] + sub_paths = [] + Dir.entries(path).each do |entry| + next if excluded.include?(entry) + + sub_path = File.join(path, entry) + + if File.directory?(sub_path) + sub_paths.push(sub_path) + elsif sub_path =~ /\A(.*)\.rb\z/ + require_path = $1 + require(require_path) + end + end + + sub_paths.each do |sub_path| + eager_load_recursive(sub_path) + end + end + end +end diff --git a/lib/mixin/start_finish_logger.rb b/lib/mixin/start_finish_logger.rb new file mode 100644 index 000000000..7bee862ac --- /dev/null +++ b/lib/mixin/start_finish_logger.rb @@ -0,0 +1,11 @@ +module Mixin + module StartFinishLogger + include ::Mixin::RailsLogger + + def log_start_finish(level, prefix) + logger.public_send(level, "#{prefix} started.") + yield + logger.public_send(level, "#{prefix} finished.") + end + end +end diff --git a/lib/sequencer.rb b/lib/sequencer.rb new file mode 100644 index 000000000..7a606eed1 --- /dev/null +++ b/lib/sequencer.rb @@ -0,0 +1,83 @@ +require 'mixin/rails_logger' +require 'mixin/start_finish_logger' + +class Sequencer + include ::Mixin::RailsLogger + include ::Mixin::StartFinishLogger + + attr_reader :sequence + + # Convenience wrapper for instant processing with the given attributes. + # + # @example + # Sequencer.process('Example::Sequence') + # + # @example + # Sequencer.process('Example::Sequence', + # parameters: { + # some: 'value', + # }, + # expecting: [:result, :answer] + # ) + # + # @return [Hash{Symbol => Object}] the final result state attributes and values + def self.process(sequence, *args) + new(sequence, *args).process + end + + # Initializes a new Sequencer instance for the given Sequence with parameters and expecting result. + # + # @example + # Sequencer.new('Example::Sequence') + # + # @example + # Sequencer.new('Example::Sequence', + # parameters: { + # some: 'value', + # }, + # expecting: [:result, :answer] + # ) + def initialize(sequence, parameters: {}, expecting: nil) + @sequence = Sequencer::Sequence.constantize(sequence) + @parameters = parameters + @expecting = expecting + + # fall back to sequence default expecting if no explicit + # expecting was given for this sequence + return if !@expecting.nil? + @expecting = @sequence.expecting + end + + # Processes the Sequence the instance was initialized with. + # + # @example + # sequence.process + # + # @return [Hash{Symbol => Object}] the final result state attributes and values + def process + log_start_finish(:info, "Sequence '#{@sequence.name}'") do + + sequence.units.each_with_index do |unit, index| + + state.process do + + log_start_finish(:info, "Sequence '#{sequence.name}' Unit '#{unit.name}' (index: #{index})") do + unit.process(state) + end + end + end + end + + state.to_h.tap do |result| + logger.debug("Returning Sequence '#{@sequence.name}' result: #{result.inspect}") + end + end + + private + + def state + @state ||= Sequencer::State.new(sequence, + parameters: @parameters, + expecting: @expecting) + end +end diff --git a/lib/sequencer/mixin/exchange/folder.rb b/lib/sequencer/mixin/exchange/folder.rb new file mode 100644 index 000000000..0740da383 --- /dev/null +++ b/lib/sequencer/mixin/exchange/folder.rb @@ -0,0 +1,18 @@ +class Sequencer + module Mixin + module Exchange + module Folder + + def self.included(base) + base.uses :ews_connection + end + + private + + def ews_folder + @ews_folder ||= ::Import::Exchange::Folder.new(ews_connection) + end + end + end + end +end diff --git a/lib/sequencer/mixin/import_job/resource_loop.rb b/lib/sequencer/mixin/import_job/resource_loop.rb new file mode 100644 index 000000000..a27e7e670 --- /dev/null +++ b/lib/sequencer/mixin/import_job/resource_loop.rb @@ -0,0 +1,37 @@ +require 'sequencer/mixin/sub_sequence' + +class Sequencer + module Mixin + module ImportJob + module ResourceLoop + include ::Sequencer::Mixin::SubSequence + + private + + def resource_sequence(sequence_name, items) + default_params = { + dry_run: import_job.dry_run, + import_job: import_job, + } + + items.each do |item| + resource_params = {} + if block_given? + resource_params = yield item + else + resource_params[:resource] = item + end + + resource_params[:resource] = resource_params[:resource].with_indifferent_access + + sub_sequence(sequence_name, + parameters: default_params.merge(resource_params)) + end + + # store possible unsaved values in result e.g. statistics + import_job.save! + end + end + end + end +end diff --git a/lib/sequencer/mixin/prefixed_constantize.rb b/lib/sequencer/mixin/prefixed_constantize.rb new file mode 100644 index 000000000..243bee274 --- /dev/null +++ b/lib/sequencer/mixin/prefixed_constantize.rb @@ -0,0 +1,43 @@ +class Sequencer + module Mixin + # Classes that extend this module need a PREFIX constant. + module PrefixedConstantize + # Returns the class for a given name String independend of the prefix. + # + # @param [String] sequence the name String for the requested class + # + # @example + # Sequencer::Sequence.constantize('ExampleSequence') + # #=> Sequencer::Sequence::ExampleSequence + # + # @example + # Sequencer::Unit.constantize('Sequencer::Unit::Example::Unit') + # #=> Sequencer::Unit::Example::Unit + # + # @return [Object] the class for the given String + def constantize(name_string) + namespace(name_string).constantize + end + + # Returns the complete class namespace for a given name String + # independend of the prefix. + # + # @param [String] sequence the name String for the requested class namespace + # + # @example + # Sequencer::Sequence.namespace('ExampleSequence') + # #=> 'Sequencer::Sequence::ExampleSequence' + # + # @example + # Sequencer::Unit.namespace('Sequencer::Unit::Example::Unit') + # #=> 'Sequencer::Unit::Example::Unit' + # + # @return [String] the class namespace for the given String + def namespace(name_string) + prefix = const_get(:PREFIX) + return name_string if name_string.start_with?(prefix) + "#{prefix}#{name_string}" + end + end + end +end diff --git a/lib/sequencer/mixin/sub_sequence.rb b/lib/sequencer/mixin/sub_sequence.rb new file mode 100644 index 000000000..75348641c --- /dev/null +++ b/lib/sequencer/mixin/sub_sequence.rb @@ -0,0 +1,9 @@ +class Sequencer + module Mixin + module SubSequence + def sub_sequence(sequence, args = {}) + Sequencer.process(sequence, args) + end + end + end +end diff --git a/lib/sequencer/sequence.rb b/lib/sequencer/sequence.rb new file mode 100644 index 000000000..906ee2ba3 --- /dev/null +++ b/lib/sequencer/sequence.rb @@ -0,0 +1,16 @@ +require 'sequencer/mixin/prefixed_constantize' + +class Sequencer + class Sequence + extend ::Sequencer::Mixin::PrefixedConstantize + + PREFIX = 'Sequencer::Sequence::'.freeze + + attr_reader :units, :expecting + + def initialize(units:, expecting: []) + @units = units + @expecting = expecting + end + end +end diff --git a/lib/sequencer/sequence/base.rb b/lib/sequencer/sequence/base.rb new file mode 100644 index 000000000..382d47baa --- /dev/null +++ b/lib/sequencer/sequence/base.rb @@ -0,0 +1,48 @@ +class Sequencer + class Sequence + class Base + + # Defines the default attributes that will get returned for the sequence. + # These can be overwritten by giving the :expecting key to a sequence process call. + # + # @example + # Sequencer::Sequence::Example.expecting + # # => [:result, :list] + # + # @return [Array] the list of expected result keys. + def self.expecting + [] + end + + # Defines the list of Units that form the sequence. The units will get + # processed in the order they are defined in. The namespaces can be + # absolute or without the `Sequencer::Unit` prefix. + # + # @example + # Sequencer::Sequence::Example.sequence + # # => ['Import::Example::Resource', 'Sequencer::Unit::Import::Model::Create', ...] + # + # @return [Array] the list of units forming the sequence. + def self.sequence + raise "Missing implementation of '#{__method__}' method for '#{name}'" + end + + # This is an internally used method that converts the defined sequence to a + # Sequencer::Units instance which has special methods. + # + # @example + # Sequencer::Sequence::Example.units + # # => + # + # @return [Object] + def self.units + Sequencer::Units.new(*sequence) + end + + # @see .units + def units + self.class.units + end + end + end +end diff --git a/lib/sequencer/sequence/exchange/folder/attributes.rb b/lib/sequencer/sequence/exchange/folder/attributes.rb new file mode 100644 index 000000000..547d86e4f --- /dev/null +++ b/lib/sequencer/sequence/exchange/folder/attributes.rb @@ -0,0 +1,17 @@ +class Sequencer + class Sequence + module Exchange + module Folder + class Attributes < Sequencer::Sequence::Base + + def self.sequence + [ + 'Exchange::Connection', + 'Exchange::Folder::Attributes', + ] + end + end + end + end + end +end diff --git a/lib/sequencer/sequence/import/exchange/attributes_examples.rb b/lib/sequencer/sequence/import/exchange/attributes_examples.rb new file mode 100644 index 000000000..6c5cdab2b --- /dev/null +++ b/lib/sequencer/sequence/import/exchange/attributes_examples.rb @@ -0,0 +1,23 @@ +class Sequencer + class Sequence + module Import + module Exchange + class AttributesExamples < Sequencer::Sequence::Base + + def self.expecting + [:attributes] + end + + def self.sequence + [ + 'Exchange::Connection', + 'Exchange::Folders::ByIds', + 'Import::Exchange::AttributeExamples', + 'Import::Exchange::AttributeMapper::AttributeExamples', + ] + end + end + end + end + end +end diff --git a/lib/sequencer/sequence/import/exchange/available_folders.rb b/lib/sequencer/sequence/import/exchange/available_folders.rb new file mode 100644 index 000000000..63c37b927 --- /dev/null +++ b/lib/sequencer/sequence/import/exchange/available_folders.rb @@ -0,0 +1,22 @@ +class Sequencer + class Sequence + module Import + module Exchange + class AvailableFolders < Sequencer::Sequence::Base + + def self.expecting + [:folders] + end + + def self.sequence + [ + 'Exchange::Connection', + 'Exchange::Folders::IdPathMap', + 'Import::Exchange::AttributeMapper::AvailableFolders', + ] + end + end + end + end + end +end diff --git a/lib/sequencer/sequence/import/exchange/folder_contact.rb b/lib/sequencer/sequence/import/exchange/folder_contact.rb new file mode 100644 index 000000000..d475920ca --- /dev/null +++ b/lib/sequencer/sequence/import/exchange/folder_contact.rb @@ -0,0 +1,33 @@ +class Sequencer + class Sequence + module Import + module Exchange + class FolderContact < Sequencer::Sequence::Base + + def self.sequence + [ + 'Import::Exchange::FolderContact::RemoteId', + 'Import::Exchange::FolderContact::Mapping', + 'Import::Common::Model::Skip::Blank::Mapped', + 'Import::Exchange::FolderContact::StaticAttributes', + 'Import::Common::Model::ExternalSync::Lookup', + 'Import::Common::Model::Associations::Extract', + 'Import::Common::User::Attributes::Downcase', + 'Import::Common::User::Email::CheckValidity', + 'Import::Common::Model::Attributes::AddByIds', + 'Import::Common::Model::Update', + 'Import::Common::Model::Create', + 'Import::Common::Model::Associations::Assign', + 'Import::Common::Model::Save', + 'Import::Common::Model::ExternalSync::Create', + 'Import::Exchange::FolderContact::HttpLog', + 'Import::Exchange::FolderContact::Statistics::Diff', + 'Import::Common::ImportJob::Statistics::Update', + 'Import::Common::ImportJob::Statistics::Store', + ] + end + end + end + end + end +end diff --git a/lib/sequencer/sequence/import/exchange/folder_contacts.rb b/lib/sequencer/sequence/import/exchange/folder_contacts.rb new file mode 100644 index 000000000..9d07f75cd --- /dev/null +++ b/lib/sequencer/sequence/import/exchange/folder_contacts.rb @@ -0,0 +1,22 @@ +class Sequencer + class Sequence + module Import + module Exchange + class FolderContacts < Sequencer::Sequence::Base + + def self.sequence + [ + 'Import::Exchange::FolderContacts::DryRunPayload', + 'Exchange::Connection', + 'Import::Exchange::FolderContacts::FolderIds', + 'Import::Exchange::FolderContacts::Sum', + 'Import::Common::ImportJob::Statistics::Update', + 'Import::Common::ImportJob::Statistics::Store', + 'Import::Exchange::FolderContacts::SubSequence', + ] + end + end + end + end + end +end diff --git a/lib/sequencer/state.rb b/lib/sequencer/state.rb new file mode 100644 index 000000000..39a7107a6 --- /dev/null +++ b/lib/sequencer/state.rb @@ -0,0 +1,272 @@ +require 'mixin/rails_logger' +require 'mixin/start_finish_logger' + +class Sequencer + class State + include ::Mixin::RailsLogger + include ::Mixin::StartFinishLogger + + def initialize(sequence, parameters: {}, expecting: nil) + @index = -1 + @units = sequence.units + @result_index = @units.count + @values = {} + + initialize_attributes(sequence.units) + initialize_parameters(parameters.with_indifferent_access) + initialize_expectations(expecting || sequence.expecting) + end + + # Stores a value for the given attribute. Value can be a regular object + # or the result of a given code block. + # The attribute gets validated against the .provides list of attributes. + # In the case than an attribute gets provided that is not declared to + # be provided an exception will be raised. + # + # @param [Symbol] attribute the attribute for which the value gets provided. + # @param [Object] value the value that should get stored for the given attribute. + # @yield [] executes the given block and takes the result as the value. + # @yieldreturn [Object] the value for the given attribute. + # + # @example + # state.provide(:sum, 3) + # + # @example + # state.provide(:sum) do + # some_value = rand(100) + # some_value * 3 + # end + # + # @raise [RuntimeError] if the attribute is not provideable from the calling Unit + # + # @return [nil] + def provide(attribute, value = nil) + if provideable?(attribute) + value = yield if block_given? + set(attribute, value) + else + value = "UNEXECUTED BLOCK: #{caller(1..1).first}" if block_given? + unprovideable_setter(attribute, value) + end + end + + # Returns the value of the given attribute. + # The attribute gets validated against the .uses list of attributes. In the + # case than an attribute gets used that is not declared to be used + # an exception will be raised. + # + # @param [Symbol] attribute the attribute for which the value is requested. + # + # @example + # state.use(:answer) + # #=> 42 + # + # @raise [RuntimeError] if the attribute is not useable from the calling Unit + # + # @return [nil] + def use(attribute) + if useable?(attribute) + get(attribute) + else + unaccessable_getter(attribute) + end + end + + # Returns the value of the given attribute. + # The attribute DOES NOT get validated against the .uses list of attributes. + # + # @param [Symbol] attribute the attribute for which the value is requested. + # + # @example + # state.optional(:answer) + # #=> 42 + # + # @example + # state.optional(:unknown) + # #=> nil + # + # @return [Object, nil] + def optional(attribute) + return get(attribute) if @attributes.known?(attribute) + logger.debug("Access to unknown optional attribute '#{attribute}'.") + nil + end + + # Checks if a value for the given attribute is provided. + # The attribute DOES NOT get validated against the .uses list of attributes. + # + # @param [Symbol] attribute the attribute which should get checked. + # + # @example + # state.provided?(:answer) + # #=> true + # + # @example + # state.provided?(:unknown) + # #=> false + # + # @return [Boolean] + def provided?(attribute) + optional(attribute) != nil + end + + # Unsets the value for the given attribute. + # The attribute gets validated against the .uses list of attributes. + # In the case than an attribute gets unset that is not declared + # to be used an exception will be raised. + # + # @param [Symbol] attribute the attribute for which the value gets unset. + # + # @example + # state.unset(:answer) + # + # @raise [RuntimeError] if the attribute is not useable from the calling Unit + # + # @return [nil] + def unset(attribute) + value = nil + if useable?(attribute) + set(attribute, value) + else + unprovideable_setter(attribute, value) + end + end + + # Handles state processing of the next Unit in the Sequence while executing + # the given block. After the Unit is processed the state will get cleaned up + # and no longer needed attribute values will get discarded. + # + # @yield [] executes the given block and handles the state changes before and afterwards. + # + # @example + # state.process do + # unit.process + # end + # + # @return [nil] + def process + @index += 1 + yield + cleanup + end + + # Handles state processing of the next Unit in the Sequence while executing + # the given block. After the Unit is processed the state will get cleaned up + # and no longer needed attribute values will get discarded. + # + # @example + # state.to_h + # #=> {"ssl_verify"=>true, "host_url"=>"ldaps://192...", ...} + # + # @return [Hash{Symbol => Object}] + def to_h + available.map { |identifier| [identifier, @values[identifier]] }.to_h + end + + private + + def available + @attributes.select do |_identifier, attribute| + @index.between?(attribute.from, attribute.to) + end.keys + end + + def unit(index = nil) + @units[index || @index] + end + + def provideable?(attribute) + unit.provides.include?(attribute) + end + + def useable?(attribute) + unit.uses.include?(attribute) + end + + def set(attribute, value) + logger.debug("Setting '#{attribute}' value (#{value.class.name}): #{value.inspect}") + @values[attribute] = value + end + + def get(attribute) + value = @values[attribute] + logger.debug("Getting '#{attribute}' value (#{value.class.name}): #{value.inspect}") + value + end + + def unprovideable_setter(attribute, value) + message = "Unprovideable attribute '#{attribute}' set with value (#{value.class.name}): '#{value}'" + logger.error(message) + raise message + end + + def unaccessable_getter(attribute) + message = "Unaccessable getter used for attribute '#{attribute}'" + logger.error(message) + raise message + end + + def initialize_attributes(units) + log_start_finish(:debug, 'Attributes lifespan initialization') do + @attributes = Sequencer::Units::Attributes.new(units.declarations) + logger.debug("Attributes lifespan: #{@attributes.inspect}") + end + end + + def initialize_parameters(parameters) + logger.debug("Initializing Sequencer::State with initial parameters: #{parameters.inspect}") + + log_start_finish(:debug, 'Attribute value provisioning check and initialization') do + + @attributes.each do |identifier, attribute| + + if !attribute.will_be_used? + logger.debug("Attribute '#{identifier}' is provided by Unit(s) but never used.") + next + end + + init_param = parameters.key?(identifier) + provided_attr = attribute.will_be_provided? + + if !init_param && !provided_attr + message = "Attribute '#{identifier}' is used in Unit '#{unit(attribute.to).name}' (index: #{attribute.to}) but is not provided or given via initial parameters." + logger.error(message) + raise message + end + + # skip if attribute is provided by an Unit but not + # an initial parameter + next if !init_param + + # update 'from' lifespan information for attribute + # since it's provided via the initial parameter + attribute.from = @index + + # set initial value + set(identifier, parameters[identifier]) + end + end + end + + def initialize_expectations(expected_attributes) + expected_attributes.each do |identifier| + logger.debug("Adding attribute '#{identifier}' to the list of expected result attributes.") + @attributes[identifier].to = @result_index + end + end + + def cleanup + log_start_finish(:info, "State cleanup of Unit #{unit.name} (index: #{@index})") do + + @attributes.delete_if do |identifier, attribute| + remove = !attribute.will_be_used? + remove ||= attribute.to <= @index + if remove && attribute.will_be_used? + logger.debug("Removing unneeded attribute '#{identifier}': #{@values[identifier]}") + end + remove + end + end + end + end +end diff --git a/lib/sequencer/unit.rb b/lib/sequencer/unit.rb new file mode 100644 index 000000000..723ef4a2e --- /dev/null +++ b/lib/sequencer/unit.rb @@ -0,0 +1,71 @@ +require 'sequencer/mixin/prefixed_constantize' + +class Sequencer + class Unit + include ::Mixin::RequiredSubPaths + extend ::Sequencer::Mixin::PrefixedConstantize + + PREFIX = 'Sequencer::Unit::'.freeze + + # Convenience wrapper for processing a single Unit. + # + # ATTENTION: This should only be used for development, testing or debugging purposes. + # There might be a check in the future to prevent using this method in other scopes. + # + # @see #initialize + # @see #process + def self.process(unit, parameters, &block) + new(unit).process(parameters, &block) + end + + # Initializes a new Sequencer::Unit for processing it. + # + # ATTENTION: This should only be used for development, testing or debugging purposes. + # There might be a check in the future to prevent using this method in other scopes. + # + # @param [String] unit the name String for the Unit that should get processed + def initialize(unit) + @unit = self.class.constantize(unit) + end + + # Processes the Sequencer::Unit that the instance was initialized with. + # + # ATTENTION: This should only be used for development, testing or debugging purposes. + # There might be a check in the future to prevent using this method in other scopes. + # + # @param [Hash{Symbol => Object}] parameters the parameters for initializing the Sequencer::State + # @yield [instance] optional block to access the Unit instance + # @yieldparam instance [Object] the Unit instance for e.g. adding expectations + def process(parameters) + @parameters = parameters + instance = @unit.new(state) + + # yield instance to apply expectations + yield instance if block_given? + + state.process do + instance.process + end + + state.to_h + end + + private + + def state + @state ||= begin + units = Sequencer::Units.new( + @unit.name + ) + + sequence = Sequencer::Sequence.new( + units: units, + expecting: @unit.provides, + ) + + Sequencer::State.new(sequence, + parameters: @parameters) + end + end + end +end diff --git a/lib/sequencer/unit/base.rb b/lib/sequencer/unit/base.rb new file mode 100644 index 000000000..cd97afe82 --- /dev/null +++ b/lib/sequencer/unit/base.rb @@ -0,0 +1,212 @@ +require 'mixin/rails_logger' + +class Sequencer + class Unit + class Base + include ::Mixin::RailsLogger + + attr_reader :state + + # Creates the class macro `uses` that allows a Unit to + # declare the attributes it will use via parameter or block. + # On the other hand it returns the declared attributes if + # called without parameters. + # + # This method can be called multiple times and will add the + # given attributes to the list. It takes care of handling + # duplicates so no uniq check is required. It's safe to use + # for inheritance structures and modules. + # + # It additionally creates a getter instance method for each declared + # attribute like e.g. attr_reader does. This allows direct access + # to an attribute via `attribute_name`. See examples. + # + # @param [Array] attributes an optional list of attributes that the Unit uses + # + # @yield [] A block returning a list of attributes + # + # @example Via regular Array parameter + # uses :instance, :action, :connection + # + # @example Via block + # uses do + # additional = method(parameter) + # [:some, additional] + # end + # + # @example Listing declared attributes + # Unit::Name.uses + # # => [:instance, :action, :connection, :some, :suprise] + # + # @example Using declared attribute in the Unit via state object + # state.use(:instance).id + # + # @example Using declared attribute in the Unit via getter + # instance.id + # + # @return [Array] the list of all declared uses of a Unit. + def self.uses(*attributes, &block) + declaration_accessor( + key: __method__, + attributes: attributes(*attributes, &block) + ) do |attribute| + use_getter(attribute) + end + end + + # Creates the class macro `provides` that allows a Unit to + # declare the attributes it will provided via parameter or block. + # On the other hand it returns the declared attributes if + # called without parameters. + # + # This method can be called multiple times and will add the + # given attributes to the list. It takes care of handling + # duplicates so no uniq check is required. It's safe to use + # for inheritance structures and modules. + # + # It additionally creates a setter instance method for each declared + # attribute like e.g. attr_writer does. This allows direct access + # to an attribute via `self.attribute_name = `. See examples. + # + # A Unit should usually not provide more than one or two attributes. + # If your Unit provides it's doing to much and should be splitted + # into multiple Units. + # + # @param [Array] attributes an optional list of attributes that the Unit provides + # + # @yield [] A block returning a list of attributes + # + # @example Via regular Array parameter + # provides :instance, :action, :connection + # + # @example Via block + # provides do + # additional = method(parameter) + # [:some, additional] + # end + # + # @example Listing declared attributes + # Unit::Name.provides + # # => [:instance, :action, :connection, :some, :suprise] + # + # @example Providing declared attribute in the Unit via state object parameter + # state.provide(:action, :created) + # + # @example Providing declared attribute in the Unit via state object block + # state.provide(:instance) do + # # ... + # instance + # end + # + # @example Providing declared attribute in the Unit via setter + # self.action = :created + # + # @return [Array] the list of all declared provides of a Unit. + def self.provides(*attributes, &block) + declaration_accessor( + key: __method__, + attributes: attributes(*attributes, &block) + ) do |attribute| + provide_setter(attribute) + end + end + + def self.attributes(*attributes) + # exectute block if given and add + # the result to the (possibly empty) + # list of given attributes + attributes.concat(yield) if block_given? + attributes + end + + # This method is the heart of the #uses and #provides method. + # It takes the declaration key and decides based on the given + # parameters if the given attributes should get stored or + # the stored values returned. + def self.declaration_accessor(key:, attributes:) + + # if no attributes were given (storing) + # return the already stored list of attributes + return declarations(key).to_a if attributes.blank? + + # loop over all given attributes and + # add them to the list of already stored + # attributes for the given declaration key + attributes.each do |attribute| + next if !declarations(key).add?(attribute) + + # yield callback if given to create + # getter or setter or whatever + yield(attribute) if block_given? + end + end + + # This method creates the convenience method + # getter for the given attribute. + def self.use_getter(attribute) + define_method(attribute) do + instance_variable_cached(attribute) do + state.use(attribute) + end + end + end + + # This method creates the convenience method + # setter for the given attribute. + def self.provide_setter(attribute) + define_method("#{attribute}=") do |value| + state.provide(attribute, value) + end + end + + # This method is the attribute store for the given declaration key. + def self.declarations(key) + instance_variable_cached("#{key}_declarations") do + declarations_initial(key) + end + end + + # This method initializes the attribute store for the given declaration key. + # It checks if a parent class already has an existing store and duplicates it + # for independent usage. Otherwise it creates a new one. + def self.declarations_initial(key) + return Set.new([]) if !superclass.respond_to?(:declarations) + superclass.send(:declarations, key).dup + end + + # This method creates an accessor to a cached instance variable for the given scope. + # It will create a new variable with the result of the given block as an initial value. + # On later calls it will return the already initialized, cached variable state. + # The variable will be created by default as a class variable. If a instance scope is + # passed it will create an instance variable instead. + def self.instance_variable_cached(key, scope: self) + cache = "@#{key}" + value = scope.instance_variable_get(cache) + return value if value + value = yield + scope.instance_variable_set(cache, value) + end + + # This method is an instance wrapper around the class method .instance_variable_cached. + # It will behave the same but passed the instance scope to create an + # cached instance variable. + def instance_variable_cached(key, &block) + self.class.instance_variable_cached(key, scope: self, &block) + end + + # This method is an convenience wrapper to create an instance + # and then directly processing it. + def self.process(*args) + new(*args).process + end + + def initialize(state) + @state = state + end + + def process + raise "Missing implementation of '#{__method__}' method for '#{self.class.name}'" + end + end + end +end diff --git a/lib/sequencer/unit/common/attribute_mapper.rb b/lib/sequencer/unit/common/attribute_mapper.rb new file mode 100644 index 000000000..8922dd190 --- /dev/null +++ b/lib/sequencer/unit/common/attribute_mapper.rb @@ -0,0 +1,28 @@ +class Sequencer + class Unit + module Common + class AttributeMapper < Sequencer::Unit::Base + + def self.map + raise "Missing implementation of '#{__method__}' method for '#{name}'" + end + + def self.uses + map.keys + end + + def self.provides + map.values + end + + def process + self.class.map.each do |original, renamed| + state.provide(renamed) do + state.use(original) + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/exchange/connection.rb b/lib/sequencer/unit/exchange/connection.rb new file mode 100644 index 000000000..fddcba8c0 --- /dev/null +++ b/lib/sequencer/unit/exchange/connection.rb @@ -0,0 +1,23 @@ +class Sequencer + class Unit + module Exchange + class Connection < Sequencer::Unit::Base + + uses :ews_config + provides :ews_connection + + def process + # check if EWS connection is already given (sub sequence) + return if state.provided?(:ews_connection) + + state.provide(:ews_connection) do + config = ews_config + config ||= ::Import::Exchange.config + + Viewpoint::EWSClient.new(config[:endpoint], config[:user], config[:password]) + end + end + end + end + end +end diff --git a/lib/sequencer/unit/exchange/folders/by_ids.rb b/lib/sequencer/unit/exchange/folders/by_ids.rb new file mode 100644 index 000000000..8a08d530f --- /dev/null +++ b/lib/sequencer/unit/exchange/folders/by_ids.rb @@ -0,0 +1,24 @@ +require 'sequencer/mixin/exchange/folder' + +class Sequencer + class Unit + module Exchange + module Folders + class ByIds < Sequencer::Unit::Base + include ::Sequencer::Mixin::Exchange::Folder + + uses :ews_folder_ids + provides :ews_folders + + def process + state.provide(:ews_folders) do + ews_folder_ids.collect do |folder_id| + ews_folder.find(folder_id) + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/exchange/folders/id_path_map.rb b/lib/sequencer/unit/exchange/folders/id_path_map.rb new file mode 100644 index 000000000..9b6d48847 --- /dev/null +++ b/lib/sequencer/unit/exchange/folders/id_path_map.rb @@ -0,0 +1,31 @@ +require 'sequencer/mixin/exchange/folder' + +class Sequencer + class Unit + module Exchange + module Folders + class IdPathMap < Sequencer::Unit::Base + include ::Sequencer::Mixin::Exchange::Folder + + provides :ews_folder_id_path_map + + def process + state.provide(:ews_folder_id_path_map) do + + ids = state.optional(:ews_folder_ids) + ids ||= [] + + ews_folder.id_folder_map.collect do |id, folder| + next if ids.present? && ids.exclude?(id) + next if folder.total_count.blank? + next if folder.total_count.zero? + + [id, ews_folder.display_path(folder)] + end.compact.to_h + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/import_job/payload/to_state.rb b/lib/sequencer/unit/import/common/import_job/payload/to_state.rb new file mode 100644 index 000000000..0c0d17324 --- /dev/null +++ b/lib/sequencer/unit/import/common/import_job/payload/to_state.rb @@ -0,0 +1,25 @@ +class Sequencer + class Unit + module Import + module Common + module ImportJob + module Payload + class ToState < Sequencer::Unit::Base + + uses :import_job + + def process + provides = self.class.provides + raise "Can't find any provides for #{self.class.name}" if provides.blank? + + provides.each do |attribute| + state.provide(attribute, import_job.payload[attribute]) + end + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/import_job/statistics/store.rb b/lib/sequencer/unit/import/common/import_job/statistics/store.rb new file mode 100644 index 000000000..33aa8a9e8 --- /dev/null +++ b/lib/sequencer/unit/import/common/import_job/statistics/store.rb @@ -0,0 +1,35 @@ +class Sequencer + class Unit + module Import + module Common + module ImportJob + module Statistics + class Store < Sequencer::Unit::Base + + uses :import_job, :statistics + + def process + # update the attribute temporarily so we can update it when: + # - the last update is more than 10 seconds in the past + # - all instances are processed but the last statistics entry is not written here. + # This will be done in the calling Unit of the executed sub sequence + import_job.result = statistics + + return if !store? + import_job.save! + end + + private + + def store? + return true if import_job.updated_at.blank? + # update every 10 seconds to reduce DB load + import_job.updated_at > Time.zone.now + 10.seconds + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/import_job/statistics/update.rb b/lib/sequencer/unit/import/common/import_job/statistics/update.rb new file mode 100644 index 000000000..57af16340 --- /dev/null +++ b/lib/sequencer/unit/import/common/import_job/statistics/update.rb @@ -0,0 +1,50 @@ +class Sequencer + class Unit + module Import + module Common + module ImportJob + module Statistics + class Update < Sequencer::Unit::Base + + uses :statistics_diff + provides :statistics + + def process + state.provide(:statistics) do + sum_deeply( + existing: statistics, + additions: statistics_diff + ) + end + + # reset diff to avoid situations where old diff gets added multiple times + state.unset(:statistics_diff) + end + + private + + def statistics + import_job = state.optional(:import_job) + return {} if import_job.nil? + import_job.result + end + + def sum_deeply(existing:, additions:) + existing.merge(additions) do |_key, oldval, newval| + if oldval.is_a?(Hash) || newval.is_a?(Hash) + sum_deeply( + existing: oldval, + additions: newval + ) + else + oldval + newval + end + end + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/import_job/sub_sequence/general.rb b/lib/sequencer/unit/import/common/import_job/sub_sequence/general.rb new file mode 100644 index 000000000..e581e1782 --- /dev/null +++ b/lib/sequencer/unit/import/common/import_job/sub_sequence/general.rb @@ -0,0 +1,66 @@ +require 'sequencer/mixin/sub_sequence' + +class Sequencer + class Unit + module Import + module Common + module ImportJob + module SubSequence + class General < Sequencer::Unit::Base + include ::Sequencer::Mixin::SubSequence + + uses :import_job + + def process + resource_sequence + end + + private + + # INFO: Cache results via `@sequence ||= ...`, if needed + def sequence + raise "Missing implementation of '#{__method__}' method for '#{self.class.name}'" + end + + # INFO: Cache results via `@resources ||= ...`, if needed + def resources + raise "Missing implementation of '#{__method__}' method for '#{self.class.name}'" + end + + def default_parameters + { + dry_run: import_job.dry_run, + import_job: import_job, + } + end + + def resource_sequence + return if resources.blank? + + defaults = default_parameters + + resources.each do |resource| + + arguments = { + parameters: defaults.merge(resource: resource) + } + + yield resource, arguments if block_given? + + arguments[:parameters][:resource] = arguments[:parameters][:resource].with_indifferent_access + + result = sub_sequence(sequence, arguments) + + processed(result) + end + end + + def processed(_result) + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/import_job/sub_sequence/resource.rb b/lib/sequencer/unit/import/common/import_job/sub_sequence/resource.rb new file mode 100644 index 000000000..7037d6e07 --- /dev/null +++ b/lib/sequencer/unit/import/common/import_job/sub_sequence/resource.rb @@ -0,0 +1,15 @@ +class Sequencer + class Unit + module Import + module Common + module ImportJob + module SubSequence + class Resource < Sequencer::Unit::Import::Common::ImportJob::SubSequence::General + uses :resource + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/mapping/flat_keys.rb b/lib/sequencer/unit/import/common/mapping/flat_keys.rb new file mode 100644 index 000000000..302671ced --- /dev/null +++ b/lib/sequencer/unit/import/common/mapping/flat_keys.rb @@ -0,0 +1,35 @@ +class Sequencer + class Unit + module Import + module Common + module Mapping + class FlatKeys < Sequencer::Unit::Base + include ::Sequencer::Unit::Import::Common::Mapping::Mixin::ProvideMapped + + uses :resource + provides :mapped + + def process + provide_mapped do + mapped + end + end + + private + + def mapped + resource_with_indifferent_access = resource.with_indifferent_access + mapping.symbolize_keys.collect do |source, local| + [local, resource_with_indifferent_access[source]] + end.to_h.with_indifferent_access + end + + def mapping + raise "Missing implementation of '#{__method__}' method for '#{self.class.name}'" + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/mapping/mixin/provide_mapped.rb b/lib/sequencer/unit/import/common/mapping/mixin/provide_mapped.rb new file mode 100644 index 000000000..544dc9727 --- /dev/null +++ b/lib/sequencer/unit/import/common/mapping/mixin/provide_mapped.rb @@ -0,0 +1,30 @@ +class Sequencer + class Unit + module Import + module Common + module Mapping + module Mixin + module ProvideMapped + + def self.included(base) + base.provides :mapped + end + + private + + def existing_mapped + @existing_mapped ||= state.optional(:mapped) || ActiveSupport::HashWithIndifferentAccess.new + end + + def provide_mapped + state.provide(:mapped) do + existing_mapped.merge(yield) + end + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/associations/assign.rb b/lib/sequencer/unit/import/common/model/associations/assign.rb new file mode 100644 index 000000000..00752ad1b --- /dev/null +++ b/lib/sequencer/unit/import/common/model/associations/assign.rb @@ -0,0 +1,59 @@ +require 'sequencer/unit/import/common/model/mixin/handle_failure' + +class Sequencer + class Unit + module Import + module Common + module Model + module Associations + class Assign < Sequencer::Unit::Base + include ::Sequencer::Unit::Import::Common::Model::Mixin::HandleFailure + + uses :instance, :associations, :instance_action, :dry_run + provides :instance_action + + def process + return if dry_run + return if instance.blank? + + instance.assign_attributes(associations) + + # execute associations check only if needed for performance reasons + return if instance_action != :unchanged + return if !changed? + state.provide(:instance_action, :changed) + rescue => e + handle_failure(e) + end + + private + + def changed? + logger.debug("Changed instance associations: #{changes.inspect}") + changes.present? + end + + def changes + @changes ||= begin + return {} if associations.blank? + associations.collect do |association, value| + before = compareable(instance.send(association)) + after = compareable(value) + next if before == after + [association, [before, after]] + end.compact.to_h.with_indifferent_access + end + end + + def compareable(value) + return nil if value.blank? + return value.sort if value.respond_to(:sort) + value + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/associations/extract.rb b/lib/sequencer/unit/import/common/model/associations/extract.rb new file mode 100644 index 000000000..40142b08e --- /dev/null +++ b/lib/sequencer/unit/import/common/model/associations/extract.rb @@ -0,0 +1,65 @@ +class Sequencer + class Unit + module Import + module Common + module Model + module Associations + class Extract < Sequencer::Unit::Base + + uses :model_class, :mapped + provides :associations + + def process + state.provide(:associations) do + associations.collect do |association| + next if !mapped.key?(association) + + # remove from the mapped values if it's an association + value = mapped.delete(association) + + # skip if we don't track them + next if tracked_associations.exclude?(association) + + [association, value] + end.compact.to_h + end + end + + private + + def associations + @associations ||= begin + associations = [] + # loop over all reflections + model_class.reflect_on_all_associations.each do |reflection| + + # refection name is something like groups or organization (singular/plural) + associations.push(reflection.name) + + # key is something like group_id or organization_id (singular) + key = reflection.klass.name.foreign_key + + # add trailing 's' to get pluralized key + reflection_name = reflection.name.to_s + if reflection_name.singularize == reflection_name + key = "#{key}s" + end + + # store _id/_ids name + associations.push(key.to_sym) + end + associations + end + end + + def tracked_associations + # track all associations by default + associations + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/attributes/add_by_ids.rb b/lib/sequencer/unit/import/common/model/attributes/add_by_ids.rb new file mode 100644 index 000000000..23ab42192 --- /dev/null +++ b/lib/sequencer/unit/import/common/model/attributes/add_by_ids.rb @@ -0,0 +1,24 @@ +class Sequencer + class Unit + module Import + module Common + module Model + module Attributes + class AddByIds < Sequencer::Unit::Base + include ::Sequencer::Unit::Import::Common::Mapping::Mixin::ProvideMapped + + def process + provide_mapped do + { + created_by_id: 1, + updated_by_id: 1, + } + end + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/attributes/check_mandatory.rb b/lib/sequencer/unit/import/common/model/attributes/check_mandatory.rb new file mode 100644 index 000000000..dbb1ba529 --- /dev/null +++ b/lib/sequencer/unit/import/common/model/attributes/check_mandatory.rb @@ -0,0 +1,32 @@ +class Sequencer + class Unit + module Import + module Common + module Model + module Attributes + class CheckMandatory < Sequencer::Unit::Base + prepend ::Sequencer::Unit::Import::Common::Model::Mixin::SkipOnProvidedInstanceAction + + uses :mapped + provides :instance_action + + def process + mandatory.each do |mapped_attribute| + next if mapped[mapped_attribute].present? + state.provide(:instance_action, :skipped) + break + end + end + + private + + def mandatory + raise "Missing implementation of '#{__method__}' method for '#{self.class.name}'" + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/attributes/remote_id.rb b/lib/sequencer/unit/import/common/model/attributes/remote_id.rb new file mode 100644 index 000000000..412fbb773 --- /dev/null +++ b/lib/sequencer/unit/import/common/model/attributes/remote_id.rb @@ -0,0 +1,32 @@ +class Sequencer + class Unit + module Import + module Common + module Model + module Attributes + class RemoteId < Sequencer::Unit::Base + include ::Sequencer::Unit::Import::Common::Model::Mixin::HandleFailure + + uses :resource + provides :remote_id + + def process + state.provide(:remote_id) do + resource.fetch(attribute) + end + rescue KeyError => e + handle_failure(e) + end + + private + + def attribute + :id + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/create.rb b/lib/sequencer/unit/import/common/model/create.rb new file mode 100644 index 000000000..8aab65fb5 --- /dev/null +++ b/lib/sequencer/unit/import/common/model/create.rb @@ -0,0 +1,25 @@ +class Sequencer + class Unit + module Import + module Common + module Model + class Create < Sequencer::Unit::Base + include ::Sequencer::Unit::Import::Common::Model::Mixin::HandleFailure + prepend ::Sequencer::Unit::Import::Common::Model::Mixin::SkipOnProvidedInstanceAction + + uses :mapped, :model_class + provides :instance, :instance_action + + def process + instance = model_class.new(mapped) + state.provide(:instance, instance) + state.provide(:instance_action, :created) + rescue => e + handle_failure(e) + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/external_sync/create.rb b/lib/sequencer/unit/import/common/model/external_sync/create.rb new file mode 100644 index 000000000..b1354a454 --- /dev/null +++ b/lib/sequencer/unit/import/common/model/external_sync/create.rb @@ -0,0 +1,29 @@ +class Sequencer + class Unit + module Import + module Common + module Model + module ExternalSync + class Create < Sequencer::Unit::Base + prepend ::Sequencer::Unit::Import::Common::Model::Mixin::SkipOnSkippedInstance + + uses :instance, :instance_action, :remote_id, :dry_run, :external_sync_source, :model_class + + def process + return if dry_run + return if instance_action != :created + + ::ExternalSync.create( + source: external_sync_source, + source_id: remote_id, + object: model_class.name, + o_id: instance.id + ) + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/external_sync/local.rb b/lib/sequencer/unit/import/common/model/external_sync/local.rb new file mode 100644 index 000000000..5d4c6f1f8 --- /dev/null +++ b/lib/sequencer/unit/import/common/model/external_sync/local.rb @@ -0,0 +1,63 @@ +require 'sequencer/unit/import/common/model/mixin/handle_failure' + +class Sequencer + class Unit + module Import + module Common + module Model + module ExternalSync + class Local < Sequencer::Unit::Base + include ::Sequencer::Unit::Import::Common::Model::Mixin::HandleFailure + prepend ::Sequencer::Unit::Import::Common::Model::Mixin::SkipOnSkippedInstance + + uses :mapped, :remote_id, :model_class, :external_sync_source, :instance_action + provides :instance + + def process + return if state.provided?(:instance) + + return if value.blank? + return if instance.blank? + + create_external_sync + + state.provide(:instance, instance) + end + + private + + def attribute + raise "Missing implementation of '#{__method__}' method for '#{self.class.name}'" + end + + def value + mapped[attribute] + end + + def instance + @instance ||= begin + model_class.where(attribute => value).find do |local| + !ExternalSync.exists?( + source: external_sync_source, + object: model_class.name, + o_id: local.id + ) + end + end + end + + def create_external_sync + ExternalSync.create( + source: external_sync_source, + source_id: remote_id, + object: import_class.name, + o_id: instance.id + ) + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/external_sync/lookup.rb b/lib/sequencer/unit/import/common/model/external_sync/lookup.rb new file mode 100644 index 000000000..e9894d794 --- /dev/null +++ b/lib/sequencer/unit/import/common/model/external_sync/lookup.rb @@ -0,0 +1,34 @@ +class Sequencer + class Unit + module Import + module Common + module Model + module ExternalSync + class Lookup < Sequencer::Unit::Base + include ::Sequencer::Unit::Import::Common::Model::Mixin::HandleFailure + prepend ::Sequencer::Unit::Import::Common::Model::Mixin::SkipOnSkippedInstance + + uses :remote_id, :model_class, :external_sync_source + provides :instance + + def process + synced_instance = ::ExternalSync.find_by( + source: external_sync_source, + source_id: remote_id, + object: model_class.name, + ) + return if !synced_instance + + state.provide(:instance) do + model_class.find(synced_instance.o_id) + end + rescue => e + handle_failure(e) + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/http_log.rb b/lib/sequencer/unit/import/common/model/http_log.rb new file mode 100644 index 000000000..811e4f383 --- /dev/null +++ b/lib/sequencer/unit/import/common/model/http_log.rb @@ -0,0 +1,50 @@ +class Sequencer + class Unit + module Import + module Common + module Model + class HttpLog < Sequencer::Unit::Base + + uses :dry_run, :instance_action, :remote_id, :mapped, :exception + + def process + return if dry_run + ::HttpLog.create( + direction: 'out', + facility: facility, + method: 'tcp', + url: "#{instance_action} -> #{remote_id}", + status: status, + ip: nil, + request: { + content: mapped, + }, + response: { + message: response + }, + created_by_id: 1, + updated_by_id: 1, + ) + end + + private + + def status + @status ||= begin + instance_action == :failed ? :failed : :success + end + end + + def response + exception ? exception.message : status + end + + def facility + raise "Missing implementation of '#{__method__}' method for '#{self.class.name}'" + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/mixin/handle_failure.rb b/lib/sequencer/unit/import/common/model/mixin/handle_failure.rb new file mode 100644 index 000000000..d2502de14 --- /dev/null +++ b/lib/sequencer/unit/import/common/model/mixin/handle_failure.rb @@ -0,0 +1,24 @@ +class Sequencer + class Unit + module Import + module Common + module Model + module Mixin + module HandleFailure + + def self.included(base) + base.provides :exception, :instance_action + end + + def handle_failure(e) + logger.error(e) + state.provide(:exception, e) + state.provide(:instance_action, :failed) + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/mixin/skip_on_provided_instance_action.rb b/lib/sequencer/unit/import/common/model/mixin/skip_on_provided_instance_action.rb new file mode 100644 index 000000000..2e9964d69 --- /dev/null +++ b/lib/sequencer/unit/import/common/model/mixin/skip_on_provided_instance_action.rb @@ -0,0 +1,22 @@ +class Sequencer + class Unit + module Import + module Common + module Model + module Mixin + module SkipOnProvidedInstanceAction + + def process + if state.provided?(:instance_action) + logger.debug("Skipping. Attribute 'instance_action' already provided.") + else + super + end + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/mixin/skip_on_skipped_instance.rb b/lib/sequencer/unit/import/common/model/mixin/skip_on_skipped_instance.rb new file mode 100644 index 000000000..8d849c2e9 --- /dev/null +++ b/lib/sequencer/unit/import/common/model/mixin/skip_on_skipped_instance.rb @@ -0,0 +1,26 @@ +class Sequencer + class Unit + module Import + module Common + module Model + module Mixin + module SkipOnSkippedInstance + + def self.prepended(base) + base.uses :instance_action + end + + def process + if instance_action == :skipped + logger.debug("Skipping. Attribute 'instance_action' is set to :skipped.") + else + super + end + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/save.rb b/lib/sequencer/unit/import/common/model/save.rb new file mode 100644 index 000000000..8dec6b8ab --- /dev/null +++ b/lib/sequencer/unit/import/common/model/save.rb @@ -0,0 +1,25 @@ +require 'sequencer/unit/import/common/model/mixin/handle_failure' + +class Sequencer + class Unit + module Import + module Common + module Model + class Save < Sequencer::Unit::Base + include ::Sequencer::Unit::Import::Common::Model::Mixin::HandleFailure + + uses :instance, :dry_run + + def process + return if dry_run + return if instance.blank? + instance.save! + rescue => e + handle_failure(e) + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/skip/blank/base.rb b/lib/sequencer/unit/import/common/model/skip/blank/base.rb new file mode 100644 index 000000000..d57a10126 --- /dev/null +++ b/lib/sequencer/unit/import/common/model/skip/blank/base.rb @@ -0,0 +1,43 @@ +require 'sequencer/unit/mixin/dynamic_attribute' + +class Sequencer + class Unit + module Import + module Common + module Model + module Skip + module Blank + class Base < Sequencer::Unit::Base + prepend ::Sequencer::Unit::Import::Common::Model::Mixin::SkipOnProvidedInstanceAction + include ::Sequencer::Unit::Mixin::DynamicAttribute + + provides :instance_action + + def process + return if !skip? + logger.debug("Skipping. Blank #{attribute} found: #{attribute_value.inspect}") + state.provide(:instance_action, :skipped) + end + + private + + def ignore + [:id] + end + + def skip? + return true if attribute_value.blank? + relevant_blank? + end + + def relevant_blank? + !attribute_value.except(*ignore).values.any?(&:present?) + end + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/skip/blank/mapped.rb b/lib/sequencer/unit/import/common/model/skip/blank/mapped.rb new file mode 100644 index 000000000..25acaad86 --- /dev/null +++ b/lib/sequencer/unit/import/common/model/skip/blank/mapped.rb @@ -0,0 +1,17 @@ +class Sequencer + class Unit + module Import + module Common + module Model + module Skip + module Blank + class Mapped < Sequencer::Unit::Import::Common::Model::Skip::Blank::Base + uses :mapped + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/skip/blank/resource.rb b/lib/sequencer/unit/import/common/model/skip/blank/resource.rb new file mode 100644 index 000000000..757ca4165 --- /dev/null +++ b/lib/sequencer/unit/import/common/model/skip/blank/resource.rb @@ -0,0 +1,17 @@ +class Sequencer + class Unit + module Import + module Common + module Model + module Skip + module Blank + class Resource < Sequencer::Unit::Import::Common::Model::Skip::Blank::Base + uses :resource + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/skip/missing_mandatory/base.rb b/lib/sequencer/unit/import/common/model/skip/missing_mandatory/base.rb new file mode 100644 index 000000000..cb327161e --- /dev/null +++ b/lib/sequencer/unit/import/common/model/skip/missing_mandatory/base.rb @@ -0,0 +1,46 @@ +require 'sequencer/unit/mixin/dynamic_attribute' + +class Sequencer + class Unit + module Import + module Common + module Model + module Skip + module MissingMandatory + class Base < Sequencer::Unit::Base + prepend ::Sequencer::Unit::Import::Common::Model::Mixin::SkipOnProvidedInstanceAction + include ::Sequencer::Unit::Mixin::DynamicAttribute + + provides :instance_action + + def process + return if !skip? + logger.debug("Skipping. Missing mandatory attributes for #{attribute}: #{attribute_value.inspect}") + state.provide(:instance_action, :skipped) + end + + private + + def mandatory + raise "Missing implementation of '#{__method__}' method for '#{self.class.name}'" + end + + def skip? + return true if attribute_value.blank? + mandatory_missing? + end + + def mandatory_missing? + values = attribute_value.fetch_values(*mandatory) + !values.any?(&:present?) + rescue KeyError => e + false + end + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/skip/missing_mandatory/resource.rb b/lib/sequencer/unit/import/common/model/skip/missing_mandatory/resource.rb new file mode 100644 index 000000000..f33bc280b --- /dev/null +++ b/lib/sequencer/unit/import/common/model/skip/missing_mandatory/resource.rb @@ -0,0 +1,17 @@ +class Sequencer + class Unit + module Import + module Common + module Model + module Skip + module MissingMandatory + class Resource < Sequencer::Unit::Import::Common::Model::Skip::MissingMandatory::Base + uses :resource + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/statistics/diff.rb b/lib/sequencer/unit/import/common/model/statistics/diff.rb new file mode 100644 index 000000000..787123b1d --- /dev/null +++ b/lib/sequencer/unit/import/common/model/statistics/diff.rb @@ -0,0 +1,21 @@ +require 'sequencer/unit/import/common/model/statistics/mixin/diff' + +class Sequencer + class Unit + module Import + module Common + module Model + module Statistics + class Diff < Sequencer::Unit::Base + include ::Sequencer::Unit::Import::Common::Model::Statistics::Mixin::Diff + + def process + state.provide(:statistics_diff, diff) + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/statistics/mixin/diff.rb b/lib/sequencer/unit/import/common/model/statistics/mixin/diff.rb new file mode 100644 index 000000000..1dd1194c7 --- /dev/null +++ b/lib/sequencer/unit/import/common/model/statistics/mixin/diff.rb @@ -0,0 +1,47 @@ +class Sequencer + class Unit + module Import + module Common + module Model + module Statistics + module Mixin + module Diff + + def self.included(base) + base.uses :instance_action + base.provides :statistics_diff + end + + private + + def actions + %i(skipped created updated unchanged failed deactivated) + end + + def diff + raise "Unknown action '#{instance_action}'" if !possible? + defaults.merge( + instance_action => 1, + sum: 1, + ) + end + + def possible? + possible_actions.include?(instance_action) + end + + def defaults + possible_actions.collect { |key| [key, 0] }.to_h + end + + def possible_actions + @possible_actions ||= actions + end + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/model/update.rb b/lib/sequencer/unit/import/common/model/update.rb new file mode 100644 index 000000000..fa0947317 --- /dev/null +++ b/lib/sequencer/unit/import/common/model/update.rb @@ -0,0 +1,56 @@ +class Sequencer + class Unit + module Import + module Common + module Model + class Update < Sequencer::Unit::Base + include ::Sequencer::Unit::Import::Common::Model::Mixin::HandleFailure + prepend ::Sequencer::Unit::Import::Common::Model::Mixin::SkipOnProvidedInstanceAction + + uses :instance, :mapped + provides :instance_action + + def process + # check if no instance is given - so we can't update it + return if !instance + + # lock the current instance for write access + instance.with_lock do + # delete since we have an update and + # the record is already created + mapped.delete(:created_by_id) + + # assign regular attributes + instance.assign_attributes(mapped) + + action = changed? ? :updated : :unchanged + state.provide(:instance_action, action) + end + rescue => e + handle_failure(e) + end + + private + + def changed? + logger.debug("Changed instance attributes: #{changes.inspect}") + changes.present? + end + + def changes + @changes ||= begin + if instance.changed? + # dry run + instance.changes + else + # live run + instance.previous_changes + end + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/user/attributes/downcase.rb b/lib/sequencer/unit/import/common/user/attributes/downcase.rb new file mode 100644 index 000000000..c5791e062 --- /dev/null +++ b/lib/sequencer/unit/import/common/user/attributes/downcase.rb @@ -0,0 +1,24 @@ +class Sequencer + class Unit + module Import + module Common + module User + module Attributes + class Downcase < Sequencer::Unit::Base + prepend ::Sequencer::Unit::Import::Common::Model::Mixin::SkipOnSkippedInstance + + uses :mapped + + def process + %i(login email).each do |attribute| + next if mapped[attribute].blank? + mapped[attribute].downcase! + end + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/common/user/email/check_validity.rb b/lib/sequencer/unit/import/common/user/email/check_validity.rb new file mode 100644 index 000000000..28f035d4f --- /dev/null +++ b/lib/sequencer/unit/import/common/user/email/check_validity.rb @@ -0,0 +1,45 @@ +class Sequencer + class Unit + module Import + module Common + module User + module Email + class CheckValidity < Sequencer::Unit::Base + prepend ::Sequencer::Unit::Import::Common::Model::Mixin::SkipOnSkippedInstance + + uses :mapped + + def process + return if mapped[:email].blank? + + # TODO: This should be done totally somewhere central + mapped[:email] = ensure_valid_email(mapped[:email]) + end + + private + + def ensure_valid_email(source) + # TODO: should get unified with User#check_email + email = extract_email(source) + return if !email + email.downcase + end + + def extract_email(source) + # Support format like "Bob Smith (bob@example.com)" + if source =~ /\((.+@.+)\)/ + source = $1 + end + + Mail::Address.new(source).address + rescue + return source if source !~ /<\s*([^>]+)/ + $1.strip + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/exchange/attribute_examples.rb b/lib/sequencer/unit/import/exchange/attribute_examples.rb new file mode 100644 index 000000000..19401609d --- /dev/null +++ b/lib/sequencer/unit/import/exchange/attribute_examples.rb @@ -0,0 +1,32 @@ +require 'sequencer/mixin/exchange/folder' + +class Sequencer + class Unit + module Import + module Exchange + class AttributeExamples < Sequencer::Unit::Base + include ::Sequencer::Mixin::Exchange::Folder + + uses :ews_folder_ids + provides :ews_attributes_examples + + def process + state.provide(:ews_attributes_examples) do + ::Import::Helper::AttributesExamples.new do |extractor| + + ews_folder_ids.collect do |folder_id| + + ews_folder.find(folder_id).items.each do |resource| + attributes = ::Import::Exchange::ItemAttributes.extract(resource) + extractor.extract(attributes) + break if extractor.enough + end + end + end.examples + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/exchange/attribute_mapper/attribute_examples.rb b/lib/sequencer/unit/import/exchange/attribute_mapper/attribute_examples.rb new file mode 100644 index 000000000..482a8cf3c --- /dev/null +++ b/lib/sequencer/unit/import/exchange/attribute_mapper/attribute_examples.rb @@ -0,0 +1,18 @@ +class Sequencer + class Unit + module Import + module Exchange + module AttributeMapper + class AttributeExamples < Sequencer::Unit::Common::AttributeMapper + + def self.map + { + ews_attributes_examples: :attributes, + } + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/exchange/attribute_mapper/available_folders.rb b/lib/sequencer/unit/import/exchange/attribute_mapper/available_folders.rb new file mode 100644 index 000000000..c1b91c187 --- /dev/null +++ b/lib/sequencer/unit/import/exchange/attribute_mapper/available_folders.rb @@ -0,0 +1,18 @@ +class Sequencer + class Unit + module Import + module Exchange + module AttributeMapper + class AvailableFolders < Sequencer::Unit::Common::AttributeMapper + + def self.map + { + ews_folder_id_path_map: :folders, + } + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/exchange/folder_contact/http_log.rb b/lib/sequencer/unit/import/exchange/folder_contact/http_log.rb new file mode 100644 index 000000000..eefb9c553 --- /dev/null +++ b/lib/sequencer/unit/import/exchange/folder_contact/http_log.rb @@ -0,0 +1,18 @@ +class Sequencer + class Unit + module Import + module Exchange + module FolderContact + class HttpLog < Import::Common::Model::HttpLog + + private + + def facility + 'EWS' + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/exchange/folder_contact/mapping.rb b/lib/sequencer/unit/import/exchange/folder_contact/mapping.rb new file mode 100644 index 000000000..076492699 --- /dev/null +++ b/lib/sequencer/unit/import/exchange/folder_contact/mapping.rb @@ -0,0 +1,27 @@ +class Sequencer + class Unit + module Import + module Exchange + module FolderContact + class Mapping < Sequencer::Unit::Import::Common::Mapping::FlatKeys + + uses :import_job + + private + + def mapping + from_import_job || ::Import::Exchange.config[:attributes] + end + + def from_import_job + return if !state.provided?(:import_job) + payload = import_job.payload + return if payload.blank? + payload[:ews_attributes] + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/exchange/folder_contact/remote_id.rb b/lib/sequencer/unit/import/exchange/folder_contact/remote_id.rb new file mode 100644 index 000000000..8cbfe713f --- /dev/null +++ b/lib/sequencer/unit/import/exchange/folder_contact/remote_id.rb @@ -0,0 +1,17 @@ +class Sequencer + class Unit + module Import + module Exchange + module FolderContact + class RemoteId < Sequencer::Unit::Import::Common::Model::Attributes::RemoteId + private + + def attribute + :item_id + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/exchange/folder_contact/static_attributes.rb b/lib/sequencer/unit/import/exchange/folder_contact/static_attributes.rb new file mode 100644 index 000000000..e7e22e026 --- /dev/null +++ b/lib/sequencer/unit/import/exchange/folder_contact/static_attributes.rb @@ -0,0 +1,19 @@ +class Sequencer + class Unit + module Import + module Exchange + module FolderContact + class StaticAttributes < Sequencer::Unit::Base + + provides :model_class, :external_sync_source + + def process + state.provide(:model_class, ::User) + state.provide(:external_sync_source, 'EWS::FolderContact') + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/exchange/folder_contact/statistics/diff.rb b/lib/sequencer/unit/import/exchange/folder_contact/statistics/diff.rb new file mode 100644 index 000000000..9ed81f27f --- /dev/null +++ b/lib/sequencer/unit/import/exchange/folder_contact/statistics/diff.rb @@ -0,0 +1,39 @@ +class Sequencer + class Unit + module Import + module Exchange + module FolderContact + module Statistics + class Diff < Sequencer::Unit::Base + include ::Sequencer::Unit::Import::Common::Model::Statistics::Mixin::Diff + + uses :ews_folder_name + + def process + state.provide(:statistics_diff) do + # remove :sum since it's already set via + # the exchange item attribte + result = diff.except(:sum) + + # build structure for a general diff + # and a folder specific sub structure + result.merge( + folders: { + ews_folder_name => result + } + ) + end + end + + private + + def actions + %i(created updated unchanged skipped failed) + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/exchange/folder_contacts/dry_run_payload.rb b/lib/sequencer/unit/import/exchange/folder_contacts/dry_run_payload.rb new file mode 100644 index 000000000..5827d25d1 --- /dev/null +++ b/lib/sequencer/unit/import/exchange/folder_contacts/dry_run_payload.rb @@ -0,0 +1,14 @@ +class Sequencer + class Unit + module Import + module Exchange + module FolderContacts + class DryRunPayload < Sequencer::Unit::Import::Common::ImportJob::Payload::ToState + + provides :ews_config, :ews_folder_ids + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/exchange/folder_contacts/folder_ids.rb b/lib/sequencer/unit/import/exchange/folder_contacts/folder_ids.rb new file mode 100644 index 000000000..5e29a9cee --- /dev/null +++ b/lib/sequencer/unit/import/exchange/folder_contacts/folder_ids.rb @@ -0,0 +1,27 @@ +require 'sequencer/mixin/exchange/folder' + +class Sequencer + class Unit + module Import + module Exchange + module FolderContacts + class FolderIds < Sequencer::Unit::Base + include ::Sequencer::Mixin::Exchange::Folder + + provides :ews_folder_ids + + def process + # check if ids are already processed + return if state.provided?(:ews_folder_ids) + + state.provide(:ews_folder_ids) do + config = ::Import::Exchange.config + config[:folders] + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/exchange/folder_contacts/sub_sequence.rb b/lib/sequencer/unit/import/exchange/folder_contacts/sub_sequence.rb new file mode 100644 index 000000000..6ddd1c970 --- /dev/null +++ b/lib/sequencer/unit/import/exchange/folder_contacts/sub_sequence.rb @@ -0,0 +1,37 @@ +require 'sequencer/mixin/exchange/folder' +require 'sequencer/mixin/import_job/resource_loop' + +class Sequencer + class Unit + module Import + module Exchange + module FolderContacts + class SubSequence < Sequencer::Unit::Base + include ::Sequencer::Mixin::Exchange::Folder + include ::Sequencer::Mixin::ImportJob::ResourceLoop + + uses :ews_folder_ids, :import_job + + def process + + ews_folder_ids.each do |folder_id| + folder = ews_folder.find(folder_id) + display_path = ews_folder.display_path(folder) + + resource_sequence('Import::Exchange::FolderContact', folder.items) do |item| + + logger.debug("Extracting attributes from Exchange item: #{item.get_all_properties!.inspect}") + + { + resource: ::Import::Exchange::ItemAttributes.extract(item), + ews_folder_name: display_path, + } + end + end + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/import/exchange/folder_contacts/sum.rb b/lib/sequencer/unit/import/exchange/folder_contacts/sum.rb new file mode 100644 index 000000000..1f73d01d4 --- /dev/null +++ b/lib/sequencer/unit/import/exchange/folder_contacts/sum.rb @@ -0,0 +1,48 @@ +require 'sequencer/mixin/exchange/folder' + +class Sequencer + class Unit + module Import + module Exchange + module FolderContacts + class Sum < Sequencer::Unit::Base + include ::Sequencer::Mixin::Exchange::Folder + + uses :ews_folder_ids + provides :statistics_diff + + def process + state.provide(:statistics_diff, diff) + end + + private + + def diff + result = { + sum: 0, + } + folder_sum_map.each do |display_path, sum| + + result[display_path] = { + sum: sum + } + + result[:sum] += sum + end + result + end + + def folder_sum_map + ews_folder_ids.collect do |folder_id| + folder = ews_folder.find(folder_id) + display_path = ews_folder.display_path(folder) + + [display_path, folder.total_count] + end.to_h + end + end + end + end + end + end +end diff --git a/lib/sequencer/unit/mixin/dynamic_attribute.rb b/lib/sequencer/unit/mixin/dynamic_attribute.rb new file mode 100644 index 000000000..e37012d90 --- /dev/null +++ b/lib/sequencer/unit/mixin/dynamic_attribute.rb @@ -0,0 +1,35 @@ +# rubocop:disable Lint/NestedMethodDefinition +class Sequencer + class Unit + module Mixin + module DynamicAttribute + + def self.included(base) + + class << base + + def inherited(base) + base.extend(Forwardable) + base.instance_delegate [:attribute] => base + end + + def attribute + @attribute ||= begin + if uses.size != 1 + raise "DynamicAttribute classes can use exactly one attribute. Found #{uses.size}." + end + uses.first + end + end + end + end + + private + + def attribute_value + @attribute_value ||= state.use(attribute) + end + end + end + end +end diff --git a/lib/sequencer/units.rb b/lib/sequencer/units.rb new file mode 100644 index 000000000..e926ead87 --- /dev/null +++ b/lib/sequencer/units.rb @@ -0,0 +1,70 @@ +require 'mixin/instance_wrapper' + +class Sequencer + class Units + include ::Enumerable + include ::Mixin::InstanceWrapper + + wrap :@units + + # Initializes a new Sequencer::Units instance with the given Units Array. + # + # @param [*Array] *units a list of Units with or without the Sequencer::Unit prefix. + # + # @example + # Sequencer::Units.new('Example::Unit', 'Sequencer::Unit::Second::Unit') + def initialize(*units) + @units = units + end + + # Required #each implementation for ::Enumerable functionality. Constantizes + # the list of units given when initialized. + # + # @example + # units.each do |unit| + # unit.process(sequencer) + # end + # + # @return [nil] + def each + @units.each do |unit| + yield constantize(unit) + end + end + + # Provides an Array of :uses and :provides declarations for each Unit. + # + # @example + # units.declarations + # #=> [{uses: [:question], provides: [:answer], ...}] + # + # @return [Array Array}>] the declarations of the Units + def declarations + collect do |unit| + { + uses: unit.uses, + provides: unit.provides, + } + end + end + + # Enables the access to an Unit class via index. + # + # @param [Integer] index the index for the requested Unit class. + # + # @example + # units[1] + # #=> Sequencer::Unit::Example + # + # @return [Object] the Unit class for the requested index + def [](index) + constantize(@units[index]) + end + + private + + def constantize(unit) + Sequencer::Unit.constantize(unit) + end + end +end diff --git a/lib/sequencer/units/attribute.rb b/lib/sequencer/units/attribute.rb new file mode 100644 index 000000000..3e716bb2b --- /dev/null +++ b/lib/sequencer/units/attribute.rb @@ -0,0 +1,30 @@ +class Sequencer + class Units + class Attribute + + attr_accessor :from, :to + + # Checks if the attribute will be provided by one or more Units. + # + # @example + # attribute.will_be_provided? + # # => true + # + # @return [Boolean] + def will_be_provided? + !from.nil? + end + + # Checks if the attribute will be used by one or more Units. + # + # @example + # attribute.will_be_used? + # # => true + # + # @return [Boolean] + def will_be_used? + !to.nil? + end + end + end +end diff --git a/lib/sequencer/units/attributes.rb b/lib/sequencer/units/attributes.rb new file mode 100644 index 000000000..f9d47aa29 --- /dev/null +++ b/lib/sequencer/units/attributes.rb @@ -0,0 +1,87 @@ +require 'mixin/instance_wrapper' + +class Sequencer + class Units + class Attributes + include ::Mixin::InstanceWrapper + + wrap :@attributes + + # Initializes the lifespan store for the attributes + # of the given Units declarations. + # + # @param [Array Array<:Symbol>}>] declarations the list of Unit declarations. + # + # @example + # declarations = [{uses: [:attribute1, ...], provides: [:result], ...}] + # attributes = Sequencer::Units::Attributes(declarations) + # + # @return [nil] + def initialize(declarations) + @declarations = declarations + + initialize_attributes + initialize_lifespan + end + + # Lists all `provides` declarations of the Units the instance was initialized with. + # + # @example + # attributes.provided + # # => [:result, ...] + # + # @return [Array] the list of all `provides` declarations + def provided + select do |_attribute, instance| + instance.will_be_provided? + end.keys + end + + # Lists all `uses` declarations of the Units the instance was initialized with. + # + # @example + # attributes.used + # # => [:attribute1, ...] + # + # @return [Array] the list of all `uses` declarations + def used + select do |_attribute, instance| + instance.will_be_used? + end.keys + end + + # Checks if the given attribute is known in the list of Unit declarations. + # + # @example + # attributes.known?(:attribute2) + # # => false + # + # @return [Boolean] + def known?(attribute) + key?(attribute) + end + + private + + def initialize_attributes + @attributes = Hash.new do |hash, key| + hash[key] = Sequencer::Units::Attribute.new + end + end + + def initialize_lifespan + @declarations.each_with_index do |unit, index| + + unit[:uses].try(:each) do |attribute| + self[attribute].to = index + end + + unit[:provides].try(:each) do |attribute| + next if self[attribute].will_be_provided? + self[attribute].from = index + end + end + end + end + end +end diff --git a/spec/lib/import/base_resource_spec.rb b/spec/lib/import/base_resource_spec.rb index f0fb77016..8b5321321 100644 --- a/spec/lib/import/base_resource_spec.rb +++ b/spec/lib/import/base_resource_spec.rb @@ -1,7 +1,5 @@ require 'rails_helper' -RSpec::Matchers.define_negated_matcher :not_change, :change - RSpec.describe Import::BaseResource do it "needs an implementation of the 'import_class' method" do diff --git a/spec/lib/import/ldap/user_spec.rb b/spec/lib/import/ldap/user_spec.rb index f49140828..14e76c5bd 100644 --- a/spec/lib/import/ldap/user_spec.rb +++ b/spec/lib/import/ldap/user_spec.rb @@ -1,8 +1,6 @@ require 'rails_helper' require 'import/ldap/user' -RSpec::Matchers.define_negated_matcher :not_change, :change - RSpec.describe Import::Ldap::User do let(:uid) { 'exampleuid' } diff --git a/spec/lib/import/otrs/state_factory_spec.rb b/spec/lib/import/otrs/state_factory_spec.rb index 205cfb892..734025fec 100644 --- a/spec/lib/import/otrs/state_factory_spec.rb +++ b/spec/lib/import/otrs/state_factory_spec.rb @@ -1,8 +1,6 @@ require 'rails_helper' require 'lib/import/transaction_factory_examples' -RSpec::Matchers.define_negated_matcher :not_change, :change - RSpec.describe Import::OTRS::StateFactory do it_behaves_like 'Import::TransactionFactory' diff --git a/spec/lib/sequencer/unit/common/attribute_mapper_spec.rb b/spec/lib/sequencer/unit/common/attribute_mapper_spec.rb new file mode 100644 index 000000000..991180335 --- /dev/null +++ b/spec/lib/sequencer/unit/common/attribute_mapper_spec.rb @@ -0,0 +1,42 @@ +require 'rails_helper' + +RSpec.describe Sequencer::Unit::Common::AttributeMapper, sequencer: :unit do + + let(:map) { + { + old_key: :new_key, + second: :new_second, + } + } + + it 'expects an implementation of the .map method' do + expect do + described_class.map + end.to raise_error(RuntimeError) + end + + it 'declares uses from map keys' do + expect(described_class).to receive(:map).and_return(map) + expect(described_class.uses).to eq(map.keys) + end + + it 'declares provides from map values' do + expect(described_class).to receive(:map).and_return(map) + expect(described_class.provides).to eq(map.values) + end + + it 'maps as configured' do + + old = { + old_key: :value, + second: :second_value, + } + + allow(described_class).to receive(:map).and_return(map) + result = process(old) + + expect(result.keys.size).to eq 2 + expect(result[:new_key]).to eq old[:old_key] + expect(result[:new_second]).to eq old[:second] + end +end diff --git a/spec/lib/sequencer/unit/import/common/mapping/flat_keys_spec.rb b/spec/lib/sequencer/unit/import/common/mapping/flat_keys_spec.rb new file mode 100644 index 000000000..0ce2aacd2 --- /dev/null +++ b/spec/lib/sequencer/unit/import/common/mapping/flat_keys_spec.rb @@ -0,0 +1,38 @@ +require 'rails_helper' + +RSpec.describe Sequencer::Unit::Import::Common::Mapping::FlatKeys, sequencer: :unit do + + it 'raises an error if mapping method is not implemented' do + expect do + process( + resource: { + remote_attribute: 'value', + } + ) + end.to raise_error(RuntimeError, /mapping/) + end + + it 'maps flat key structures' do + + parameters = { + resource: { + remote_attribute: 'value', + } + } + + mapping = { + remote_attribute: :local_attribute + } + + provided = process(parameters) do |instance| + expect(instance).to receive(:mapping).and_return(mapping) + end + + expect(provided).to eq( + mapped: { + 'local_attribute' => 'value', + } + ) + expect(provided[:mapped]).to be_a(ActiveSupport::HashWithIndifferentAccess) + end +end diff --git a/spec/models/scheduler_spec.rb b/spec/models/scheduler_spec.rb index 6f9302e65..7432186d0 100644 --- a/spec/models/scheduler_spec.rb +++ b/spec/models/scheduler_spec.rb @@ -1,7 +1,5 @@ require 'rails_helper' -RSpec::Matchers.define_negated_matcher :not_change, :change - RSpec.describe Scheduler do before do diff --git a/spec/support/negated_matchers.rb b/spec/support/negated_matchers.rb new file mode 100644 index 000000000..93f5b66e5 --- /dev/null +++ b/spec/support/negated_matchers.rb @@ -0,0 +1 @@ +RSpec::Matchers.define_negated_matcher :not_change, :change diff --git a/spec/support/sequencer.rb b/spec/support/sequencer.rb new file mode 100644 index 000000000..bbbfa9f8b --- /dev/null +++ b/spec/support/sequencer.rb @@ -0,0 +1,19 @@ +module SequencerUnit + + def process(parameters, &block) + Sequencer::Unit.process(described_class.name, parameters, &block) + end +end + +module SequencerSequence + + def process(parameters = {}) + Sequencer.process(described_class.name, + parameters: parameters) + end +end + +RSpec.configure do |config| + config.include SequencerUnit, sequencer: :unit + config.include SequencerSequence, sequencer: :sequence +end diff --git a/spec/support/system_init.rb b/spec/support/system_init.rb index 2ff4deb8c..3b4bcec18 100644 --- a/spec/support/system_init.rb +++ b/spec/support/system_init.rb @@ -1,11 +1,15 @@ RSpec.configure do |config| config.before(:suite) do - FactoryGirl.create(:user, - login: 'admin', - firstname: 'Admin', - lastname: 'Admin', - email: 'admin@example.com', - password: 'admin', - roles: [Role.lookup(name: 'Admin')],) + + email = 'admin@example.com' + if !::User.exists?(email: email) + FactoryGirl.create(:user, + login: 'admin', + firstname: 'Admin', + lastname: 'Admin', + email: email, + password: 'admin', + roles: [Role.lookup(name: 'Admin')],) + end end end