Fixes #3142 - Import archive mailbox.

.rubocop/todo.rspec.yml

@@ -695,6 +695,7 @@ RSpec/NestedGroups:
     - 'spec/system/manage/organizations_spec.rb'
     - 'spec/system/ticket/create_spec.rb'
     - 'spec/system/ticket/zoom_spec.rb'
+    - 'spec/models/channel/filter/import_archive_spec.rb'
 
 RSpec/RepeatedDescription:
   Exclude:

app/assets/javascripts/app/controllers/_channel/email.coffee

@@ -656,15 +656,8 @@ class App.ChannelEmailAccountWizard extends App.WizardModal
               @account[key] = value
 
           if data.content_messages && data.content_messages > 0 && (!@account['inbound']['options'] || @account['inbound']['options']['keep_on_server'] isnt true)
-            message = App.i18n.translateContent('We have already found %s email(s) in your mailbox. Zammad will move it all from your mailbox into Zammad.', data.content_messages)
+            @probeInboundMessagesFound(data, true)
-            @$('.js-inbound-acknowledge .js-message').html(message)
+            @probeInboundArchive(data)
-            @$('.js-inbound-acknowledge .js-back').attr('data-slide', 'js-intro')
-            @$('.js-inbound-acknowledge .js-next').attr('data-slide', '')
-            @$('.js-inbound-acknowledge .js-next').unbind('click.verify').bind('click.verify', (e) =>
-              e.preventDefault()
-              @verify(@account)
-            )
-            @showSlide('js-inbound-acknowledge')
           else
             @verify(@account)
 
@@ -713,11 +706,8 @@ class App.ChannelEmailAccountWizard extends App.WizardModal
           @account.inbound = params
 
           if data.content_messages && data.content_messages > 0 && (!@account['inbound']['options'] || @account['inbound']['options']['keep_on_server'] isnt true)
-            message = App.i18n.translateContent('We have already found %s email(s) in your mailbox. Zammad will move it all from your mailbox into Zammad.', data.content_messages)
+            @probeInboundMessagesFound(data)
-            @$('.js-inbound-acknowledge .js-message').html(message)
+            @probeInboundArchive(data)
-            @$('.js-inbound-acknowledge .js-back').attr('data-slide', 'js-inbound')
-            @$('.js-inbound-acknowledge .js-next').unbind('click.verify')
-            @showSlide('js-inbound-acknowledge')
           else
             @showSlide('js-outbound')
 
@@ -744,6 +734,65 @@ class App.ChannelEmailAccountWizard extends App.WizardModal
         @enable(e)
     )
 
+  probeInboundMessagesFound: (data, verify) =>
+    message = App.i18n.translateContent('We have already found %s email(s) in your mailbox. Zammad will move it all from your mailbox into Zammad.', data.content_messages)
+    @$('.js-inbound-acknowledge .js-messageFound').html(message)
+
+    if !verify
+      @$('.js-inbound-acknowledge .js-back').attr('data-slide', 'js-inbound')
+      @$('.js-inbound-acknowledge .js-next').unbind('click.verify')
+    else
+      @$('.js-inbound-acknowledge .js-back').attr('data-slide', 'js-intro')
+      @$('.js-inbound-acknowledge .js-next').attr('data-slide', '')
+      @$('.js-inbound-acknowledge .js-next').unbind('click.verify').bind('click.verify', (e) =>
+        e.preventDefault()
+        @verify(@account)
+      )
+    @showSlide('js-inbound-acknowledge')
+
+  probeInboundArchive: (data) =>
+    if data.archive_possible isnt true
+      @$('.js-archiveMessage').addClass('hide')
+      return
+
+    @$('.js-archiveMessage').removeClass('hide')
+    message = App.i18n.translateContent('In addition, we have found emails in your mailbox that are older than %s weeks. You can import such emails as an "archive", which means that no notifications are sent and the tickets have the status "closed". However, you can find them in Zammad anytime using the search function.', data.archive_week_range)
+    @$('.js-inbound-acknowledge .js-archiveMessageCount').html(message)
+
+    configureAttributesAcknowledge = [
+      {
+        name: 'archive'
+        tag: 'boolean'
+        null: true
+        default: no
+        options: {
+          true: 'archive'
+          false: 'regular'
+        }
+        translate: true
+      },
+    ]
+
+    new App.ControllerForm(
+      elReplace: @$('.js-importTypeSelect'),
+      model:
+        configure_attributes: configureAttributesAcknowledge
+        className: ''
+      noFieldset: true
+    )
+    @$('.js-importTypeSelect select[name=archive]').on('change', (e) =>
+      value                      = $(e.target).val()
+      @account.inbound         ||= {}
+      @account.inbound.options ||= {}
+      if value is 'true'
+        @account.inbound.options.archive        = true
+        @account.inbound.options.archive_before = (new Date()).toISOString()
+      else
+        delete @account.inbound.options.archive
+        delete @account.inbound.options.archive_before
+    )
+    @$('.js-importTypeSelect select[name=archive]').trigger('change')
+
   probleOutbound: (e) =>
     e.preventDefault()
 

app/assets/javascripts/app/controllers/getting_started.coffee

@@ -769,15 +769,8 @@ class ChannelEmail extends App.WizardFullScreen
               @account[key] = value
 
           if data.content_messages && data.content_messages > 0 && (!@account['inbound']['options'] || @account['inbound']['options']['keep_on_server'] isnt true)
-            message = App.i18n.translateContent('We have already found %s email(s) in your mailbox. Zammad will move it all from your mailbox into Zammad.', data.content_messages)
+            @probeInboundMessagesFound(data, true)
-            @$('.js-inbound-acknowledge .js-message').html(message)
+            @probeInboundArchive(data)
-            @$('.js-inbound-acknowledge .js-back').attr('data-slide', 'js-intro')
-            @$('.js-inbound-acknowledge .js-next').attr('data-slide', '')
-            @$('.js-inbound-acknowledge .js-next').unbind('click.verify').bind('click.verify', (e) =>
-              e.preventDefault()
-              @verify(@account)
-            )
-            @showSlide('js-inbound-acknowledge')
           else
             @verify(@account)
 
@@ -818,11 +811,8 @@ class ChannelEmail extends App.WizardFullScreen
           @account.inbound = params
 
           if data.content_messages && data.content_messages > 0 && (!@account['inbound']['options'] || @account['inbound']['options']['keep_on_server'] isnt true)
-            message = App.i18n.translateContent('We have already found %s emails in your mailbox. Zammad will move it all from your mailbox into Zammad.', data.content_messages)
+            @probeInboundMessagesFound(data, true)
-            @$('.js-inbound-acknowledge .js-message').html(message)
+            @probeInboundArchive(data)
-            @$('.js-inbound-acknowledge .js-back').attr('data-slide', 'js-inbound')
-            @$('.js-inbound-acknowledge .js-next').unbind('click.verify')
-            @showSlide('js-inbound-acknowledge')
           else
             @showSlide('js-outbound')
 
@@ -848,6 +838,65 @@ class ChannelEmail extends App.WizardFullScreen
         @enable(e)
     )
 
+  probeInboundMessagesFound: (data, verify) =>
+    message = App.i18n.translateContent('We have already found %s email(s) in your mailbox. Zammad will move it all from your mailbox into Zammad.', data.content_messages)
+    @$('.js-inbound-acknowledge .js-messageFound').html(message)
+
+    if !verify
+      @$('.js-inbound-acknowledge .js-back').attr('data-slide', 'js-inbound')
+      @$('.js-inbound-acknowledge .js-next').unbind('click.verify')
+    else
+      @$('.js-inbound-acknowledge .js-back').attr('data-slide', 'js-intro')
+      @$('.js-inbound-acknowledge .js-next').attr('data-slide', '')
+      @$('.js-inbound-acknowledge .js-next').unbind('click.verify').bind('click.verify', (e) =>
+        e.preventDefault()
+        @verify(@account)
+      )
+    @showSlide('js-inbound-acknowledge')
+
+  probeInboundArchive: (data) =>
+    if data.archive_possible isnt true
+      @$('.js-archiveMessage').addClass('hide')
+      return
+
+    @$('.js-archiveMessage').removeClass('hide')
+    message = App.i18n.translateContent('In addition, we have found emails in your mailbox that are older than %s weeks. You can import such emails as an "archive", which means that no notifications are sent and the tickets have the status "closed". However, you can find them in Zammad anytime using the search function.', data.archive_week_range)
+    @$('.js-inbound-acknowledge .js-archiveMessageCount').html(message)
+
+    configureAttributesAcknowledge = [
+      {
+        name: 'archive'
+        tag: 'boolean'
+        null: true
+        default: no
+        options: {
+          true: 'archive'
+          false: 'regular'
+        }
+        translate: true
+      },
+    ]
+
+    new App.ControllerForm(
+      elReplace: @$('.js-importTypeSelect'),
+      model:
+        configure_attributes: configureAttributesAcknowledge
+        className: ''
+      noFieldset: true
+    )
+    @$('.js-importTypeSelect select[name=archive]').on('change', (e) =>
+      value                      = $(e.target).val()
+      @account.inbound         ||= {}
+      @account.inbound.options ||= {}
+      if value is 'true'
+        @account.inbound.options.archive        = true
+        @account.inbound.options.archive_before = (new Date()).toISOString()
+      else
+        delete @account.inbound.options.archive
+        delete @account.inbound.options.archive_before
+    )
+    @$('.js-importTypeSelect select[name=archive]').trigger('change')
+
   probleOutbound: (e) =>
     e.preventDefault()
 

app/assets/javascripts/app/views/channel/email_account_wizard.jst.eco

@@ -108,7 +108,23 @@
     <div class="modal-body">
       <div class="wizard-body vertical justified">
         <div class="alert alert--danger hide" role="alert"></div>
-        <p class="js-message"><%- @T('We have already found %s email(s) in your mailbox. Zammad will move it all from your mailbox into Zammad.', 'x') %></p>
+        <p class="js-messageFound"><%- @T('We have already found %s email(s) in your mailbox. Zammad will move it all from your mailbox into Zammad.', 'x') %></p>
+
+        <div class="js-archiveMessage">
+          <p class="js-archiveMessageCount"><%- @T('In addition, we have found emails in your mailbox that are older than %s weeks. You can import such emails as an "archive", which means that no notifications are sent and the tickets have the status "closed". However, you can find them in Zammad anytime using the search function. ', 'x') %></p>
+
+          <p><%- @T('Should the emails from this mailbox be imported as an archive or as regular emails?') %></p>
+
+          <ul>
+            <li><%- @T('Import as archive: |No notifications are sent|, the |tickets are closed| and timestamps are removed. You can still find them in Zammad using the search.') %></li>
+            <li><%- @T('Import as regular: |Notifications are sent| and the |tickets are open| - you can find the tickets in the overview of open tickets.') %></li>
+          </ul>
+
+          <p class="js-importType">
+            Import as: <span class="js-importTypeSelect"></span>
+          </p>
+        </div>
+        <div class="inbound-acknowledge-settings"></div>
       </div>
     </div>
     <div class="modal-footer">

app/assets/javascripts/app/views/getting_started/email.jst.eco

@@ -81,7 +81,23 @@
       <h2><%- @T('Email Inbound') %></h2>
       <div class="wizard-body vertical justified">
         <div class="alert alert--danger hide" role="alert"></div>
-        <p class="js-message"><%- @T('We have already found %s email(s) in your mailbox. Zammad will move it all from your mailbox into Zammad.', 'x') %></p>
+        <p class="js-messageFound"><%- @T('We have already found %s email(s) in your mailbox. Zammad will move it all from your mailbox into Zammad.', 'x') %></p>
+
+        <div class="js-archiveMessage">
+          <p class="js-archiveMessageCount"><%- @T('In addition, we have found emails in your mailbox that are older than %s weeks. You can import such emails as an "archive", which means that no notifications are sent and the tickets have the status "closed". However, you can find them in Zammad anytime using the search function. ', 'x') %></p>
+
+          <p><%- @T('Should the emails from this mailbox be imported as an archive or as regular emails?') %></p>
+
+          <ul>
+            <li><%- @T('Import as archive: |No notifications are sent|, the |tickets are closed| and timestamps are removed. You can still find them in Zammad using the search.') %></li>
+            <li><%- @T('Import as regular: |Notifications are sent| and the |tickets are open| - you can find the tickets in the overview of open tickets.') %></li>
+          </ul>
+
+          <p class="js-importType">
+            Import as: <span class="js-importTypeSelect"></span>
+          </p>
+        </div>
+        <div class="inbound-acknowledge-settings"></div>
       </div>
       <div class="wizard-controls center">
         <a class="btn btn--text btn--secondary js-goToSlide js-back" data-slide="js-intro"><%- @T('Go Back') %></a>

app/models/channel/driver/imap.rb

@@ -174,10 +174,45 @@ example
       if content_messages >= content_max_check
         content_messages = message_ids.count
       end
+
+      archive_possible   = false
+      archive_check      = 0
+      archive_max_check  = 500
+      archive_days_range = 14
+      archive_week_range = archive_days_range / 7
+      message_ids.reverse_each do |message_id|
+        message_meta = nil
+        timeout(1.minute) do
+          message_meta = @imap.fetch(message_id, ['RFC822.HEADER'])[0]
+        end
+
+        headers = self.class.extract_rfc822_headers(message_meta)
+        next if messages_is_verify_message?(headers)
+        next if messages_is_ignore_message?(headers)
+        next if headers['Date'].blank?
+
+        archive_check += 1
+        break if archive_check >= archive_max_check
+
+        begin
+          date = Time.zone.parse(headers['Date'])
+        rescue => e
+          Rails.logger.error e
+          next
+        end
+        break if date >= Time.zone.now - archive_days_range.days
+
+        archive_possible = true
+
+        break
+      end
+
       disconnect
       return {
         result:             'ok',
         content_messages:   content_messages,
+        archive_possible:   archive_possible,
+        archive_week_range: archive_week_range,
       }
     end
 

app/models/channel/email_parser.rb

@@ -142,6 +142,11 @@ returns
 
     # run postmaster pre filter
     UserInfo.current_user_id = 1
+
+    # set interface handle
+    original_interface_handle = ApplicationHandleInfo.current
+    transaction_params = { interface_handle: "#{original_interface_handle}.postmaster", disable: [] }
+
     filters = {}
     Setting.where(area: 'Postmaster::PreFilter').order(:name).each do |setting|
       filters[setting.name] = Setting.get(setting.name).constantize
@@ -149,7 +154,7 @@ returns
     filters.each do |key, backend|
       Rails.logger.debug { "run postmaster pre filter #{key}: #{backend}" }
       begin
-        backend.run(channel, mail)
+        backend.run(channel, mail, transaction_params)
       rescue => e
         Rails.logger.error "can't run postmaster pre filter #{key}: #{backend}"
         Rails.logger.error e.inspect
@@ -163,15 +168,12 @@ returns
       return
     end
 
-    # set interface handle
-    original_interface_handle = ApplicationHandleInfo.current
-
     ticket       = nil
     article      = nil
     session_user = nil
 
     # use transaction
-    Transaction.execute(interface_handle: "#{original_interface_handle}.postmaster") do
+    Transaction.execute(transaction_params) do
 
       # get sender user
       session_user_id = mail[:'x-zammad-session-user-id']

app/models/channel/filter/auto_response_check.rb

@@ -2,7 +2,7 @@
 
 module Channel::Filter::AutoResponseCheck
 
-  def self.run(_channel, mail)
+  def self.run(_channel, mail, _transaction_params)
 
     # if header is available, do not generate auto response
     mail[ :'x-zammad-send-auto-response' ] = false

app/models/channel/filter/bounce_delivery_permanent_failed.rb

@@ -2,7 +2,7 @@
 
 module Channel::Filter::BounceDeliveryPermanentFailed
 
-  def self.run(_channel, mail)
+  def self.run(_channel, mail, _transaction_params)
 
     return if !mail[:mail_instance]
     return if !mail[:mail_instance].bounced?

app/models/channel/filter/bounce_delivery_temporary_failed.rb

@@ -2,7 +2,7 @@
 
 module Channel::Filter::BounceDeliveryTemporaryFailed
 
-  def self.run(_channel, mail)
+  def self.run(_channel, mail, _transaction_params)
     return if !mail[:mail_instance]
     return if !mail[:attachments]
     return if mail[:mail_instance].action != 'delayed'

app/models/channel/filter/bounce_follow_up_check.rb

@@ -2,7 +2,7 @@
 
 module Channel::Filter::BounceFollowUpCheck
 
-  def self.run(_channel, mail)
+  def self.run(_channel, mail, _transaction_params)
 
     return if !mail[:mail_instance]
     return if !mail[:mail_instance].bounced?

app/models/channel/filter/database.rb

@@ -3,7 +3,7 @@
 # process all database filter
 module Channel::Filter::Database
 
-  def self.run(_channel, mail)
+  def self.run(_channel, mail, _transaction_params)
 
     # process postmaster filter
     filters = PostmasterFilter.where(active: true, channel: 'email').order(:name, :created_at)

app/models/channel/filter/follow_up_check.rb

@@ -2,7 +2,7 @@
 
 module Channel::Filter::FollowUpCheck
 
-  def self.run(_channel, mail)
+  def self.run(_channel, mail, _transaction_params)
 
     return if mail[:'x-zammad-ticket-id']
 
@@ -55,32 +55,7 @@ module Channel::Filter::FollowUpCheck
     end
 
     # get ticket# from references
-    if setting.include?('references') || (mail[:'x-zammad-is-auto-response'] == true || Setting.get('ticket_hook_position') == 'none')
+    return true if ( setting.include?('references') || (mail[:'x-zammad-is-auto-response'] == true || Setting.get('ticket_hook_position') == 'none') ) && follow_up_by_md5(mail)
-
-      # get all references 'References' + 'In-Reply-To'
-      references = ''
-      if mail[:references]
-        references += mail[:references]
-      end
-      if mail[:'in-reply-to']
-        if references != ''
-          references += ' '
-        end
-        references += mail[:'in-reply-to']
-      end
-      if references != ''
-        message_ids = references.split(/\s+/)
-        message_ids.each do |message_id|
-          message_id_md5 = Digest::MD5.hexdigest(message_id)
-          article = Ticket::Article.where(message_id_md5: message_id_md5).order('created_at DESC, id DESC').limit(1).first
-          next if !article
-
-          Rails.logger.debug { "Follow-up for '##{article.ticket.number}' in references." }
-          mail[:'x-zammad-ticket-id'] = article.ticket_id
-          return true
-        end
-      end
-    end
 
     # get ticket# from references current email has same subject as initial article
     if mail[:subject].present?
@@ -125,4 +100,32 @@ module Channel::Filter::FollowUpCheck
 
     true
   end
+
+  def self.mail_references(mail)
+    references = []
+    %i[references in-reply-to].each do |key|
+      next if mail[key].blank?
+
+      references.push(mail[key])
+    end
+    references.join(' ')
+  end
+
+  def self.message_id_article(message_id)
+    message_id_md5 = Digest::MD5.hexdigest(message_id)
+    Ticket::Article.where(message_id_md5: message_id_md5).order('created_at DESC, id DESC').limit(1).first
+  end
+
+  def self.follow_up_by_md5(mail)
+    return if mail[:'x-zammad-ticket-id']
+
+    mail_references(mail).split(/\s+/).each do |message_id|
+      article = message_id_article(message_id)
+      next if article.blank?
+
+      Rails.logger.debug "Follow up for '##{article.ticket.number}' in references."
+      mail[:'x-zammad-ticket-id'] = article.ticket_id
+      return true
+    end
+  end
 end

app/models/channel/filter/follow_up_merged.rb

@@ -2,7 +2,7 @@
 
 module Channel::Filter::FollowUpMerged
 
-  def self.run(_channel, mail)
+  def self.run(_channel, mail, _transaction_params)
     return if mail[:'x-zammad-ticket-id'].blank?
 
     ticket = Ticket.find_by(id: mail[:'x-zammad-ticket-id'])

app/models/channel/filter/follow_up_possible_check.rb

@@ -2,7 +2,7 @@
 
 module Channel::Filter::FollowUpPossibleCheck
 
-  def self.run(_channel, mail)
+  def self.run(_channel, mail, _transaction_params)
     ticket_id = mail[:'x-zammad-ticket-id']
     return true if !ticket_id
 

app/models/channel/filter/identify_sender.rb

@@ -2,7 +2,7 @@
 
 module Channel::Filter::IdentifySender
 
-  def self.run(_channel, mail)
+  def self.run(_channel, mail, _transaction_params)
 
     customer_user_id = mail[ :'x-zammad-ticket-customer_id' ]
     customer_user = nil

app/models/channel/filter/import_archive.rb

@@ -0,0 +1,95 @@
+# Copyright (C) 2012-2016 Zammad Foundation, http://zammad-foundation.org/
+
+module Channel::Filter::ImportArchive
+
+  def self.run(channel, mail, transaction_params)
+    return if !import_channel?(channel, mail)
+
+    # set ignore if already imported
+    message_id = mail[:'message-id']
+    return if !message_id
+
+    # check if we already have imported this message
+    message_id_md5 = Digest::MD5.hexdigest(message_id)
+    if Ticket::Article.exists?(message_id_md5: message_id_md5)
+      mail[:'x-zammad-ignore'] = true
+      return true
+    end
+
+    # set create time if given in email
+    overwrite_created_at(mail)
+
+    # do not send auto responses
+    skip_auto_response(mail)
+
+    # set ticket to closed
+    ticket_closed(mail)
+
+    # disable notifications and trigger
+    disable_notifications(transaction_params)
+
+    # find possible follow up ticket by mail references
+    # we need this check here because in the follow up filter
+    # this check is based on settings and we want to make sure
+    # that we always check the ticket id based on the mail headers.
+    Channel::Filter::FollowUpCheck.follow_up_by_md5(mail)
+
+    true
+  end
+
+  def self.import_channel?(channel, mail)
+    return false if !mail[:date]
+
+    options = channel_options(channel)
+    return false if options[:archive] != true
+    return false if !import_channel_date_range?(channel, mail)
+
+    true
+  end
+
+  def self.import_channel_date_range?(channel, mail)
+    options = channel_options(channel)
+    return false if options[:archive_before].present? && options[:archive_before].to_date < mail[:date]
+    return false if options[:archive_till].present? && options[:archive_till].to_date < Time.now.utc
+
+    true
+  end
+
+  def self.message_id?(mail)
+    return if !mail[:'message-id']
+
+    true
+  end
+
+  def self.overwrite_created_at(mail)
+    mail[:'x-zammad-ticket-created_at'] = mail[:date]
+    mail[:'x-zammad-article-created_at'] = mail[:date]
+  end
+
+  def self.skip_auto_response(mail)
+    mail[:'x-zammad-is-auto-response'] = true
+  end
+
+  def self.ticket_closed(mail)
+    closed_state = Ticket::State.by_category(:closed).first
+    mail[:'x-zammad-ticket-state_id'] = closed_state.id
+    mail[:'x-zammad-ticket-followup-state_id'] = closed_state.id
+  end
+
+  def self.disable_notifications(transaction_params)
+    transaction_params[:disable] += %w[
+      Transaction::Notification
+      Transaction::Slack
+      Transaction::Trigger
+    ]
+  end
+
+  def self.channel_options(channel)
+    if channel.instance_of?(Channel)
+      return channel.options.dig(:inbound, :options) || {}
+    end
+
+    channel.dig(:options, :inbound, :options) || {}
+  end
+
+end

app/models/channel/filter/monitoring_base.rb

@@ -11,7 +11,7 @@ class Channel::Filter::MonitoringBase
   # Nagios
   # https://github.com/NagiosEnterprises/nagioscore/blob/754218e67653929a58938b99ef6b6039b6474fe4/sample-config/template-object/commands.cfg.in#L35
 
-  def self.run(_channel, mail)
+  def self.run(_channel, mail, _transaction_params)
     integration = integration_name
     return if !Setting.get("#{integration}_integration")
 

app/models/channel/filter/out_of_office_check.rb

@@ -2,7 +2,7 @@
 
 module Channel::Filter::OutOfOfficeCheck
 
-  def self.run(_channel, mail)
+  def self.run(_channel, mail, _transaction_params)
 
     mail[ :'x-zammad-out-of-office' ] = false
 

app/models/channel/filter/own_notification_loop_detection.rb

@@ -2,7 +2,7 @@
 
 module Channel::Filter::OwnNotificationLoopDetection
 
-  def self.run(_channel, mail)
+  def self.run(_channel, mail, _transaction_params)
 
     message_id = mail[:'message-id']
     return if !message_id

app/models/channel/filter/reply_to_based_sender.rb

@@ -2,7 +2,7 @@
 
 module Channel::Filter::ReplyToBasedSender
 
-  def self.run(_channel, mail)
+  def self.run(_channel, mail, _transaction_params)
 
     reply_to = mail[:'reply-to']
     return if reply_to.blank?

app/models/channel/filter/secure_mailing.rb

@@ -2,7 +2,7 @@
 
 module Channel::Filter::SecureMailing
 
-  def self.run(_channel, mail)
+  def self.run(_channel, mail, _transaction_params)
     ::SecureMailing.incoming(mail)
   end
 end

app/models/channel/filter/sender_is_system_address.rb

@@ -2,7 +2,7 @@
 
 module Channel::Filter::SenderIsSystemAddress
 
-  def self.run(_channel, mail)
+  def self.run(_channel, mail, _transaction_params)
 
     # if attributes already set by header
     return if mail[:'x-zammad-ticket-create-article-sender']

app/models/channel/filter/service_now_check.rb

@@ -3,7 +3,7 @@
 module Channel::Filter::ServiceNowCheck
 
   # This filter will run pre and post
-  def self.run(_channel, mail, ticket = nil, _article = nil, _session_user = nil)
+  def self.run(_channel, mail, ticket_or_transaction_params = nil, _article = nil, _session_user = nil)
     return if mail['x-servicenow-generated'].blank?
 
     source_id = self.source_id(subject: mail[:subject])
@@ -13,7 +13,7 @@ module Channel::Filter::ServiceNowCheck
     return if source_name.blank?
 
     # check if we can followup by existing service now relation
-    if ticket.blank?
+    if ticket_or_transaction_params.blank? || ticket_or_transaction_params.is_a?(Hash)
       from_sync_entry(
         mail:        mail,
         source_name: source_name,
@@ -22,7 +22,7 @@ module Channel::Filter::ServiceNowCheck
       return
     end
 
-    ExternalSync.create_with(source_id: source_id).find_or_create_by(source: source_name, object: 'Ticket', o_id: ticket.id)
+    ExternalSync.create_with(source_id: source_id).find_or_create_by(source: source_name, object: 'Ticket', o_id: ticket_or_transaction_params.id)
   end
 
 =begin

app/models/channel/filter/trusted.rb

@@ -3,7 +3,7 @@
 # delete all X-Zammad header if channel is not trusted
 module Channel::Filter::Trusted
 
-  def self.run(channel, mail)
+  def self.run(channel, mail, _transaction_params)
 
     # check if trust x-headers
     if !trusted(channel)
@@ -29,7 +29,7 @@ module Channel::Filter::Trusted
 
   def self.trusted(channel)
     return true if channel[:trusted]
-    return true if channel.instance_of?(Channel) && channel.options[:inbound][:trusted]
+    return true if channel.instance_of?(Channel) && channel.options[:inbound] && channel.options[:inbound][:trusted]
 
     false
   end

db/migrate/20190419000001_setting_add_import_archive.rb

@@ -0,0 +1,17 @@
+class SettingAddImportArchive < ActiveRecord::Migration[5.1]
+  def change
+
+    # return if it's a new setup
+    return if !Setting.exists?(name: 'system_init_done')
+
+    Setting.create_if_not_exists(
+      title:       'Define postmaster filter.',
+      name:        '0018_postmaster_import_archive',
+      area:        'Postmaster::PreFilter',
+      description: 'Define postmaster filter to import archive mailboxes.',
+      options:     {},
+      state:       'Channel::Filter::ImportArchive',
+      frontend:    false
+    )
+  end
+end

db/seeds/settings.rb

@@ -3374,6 +3374,15 @@ Setting.create_if_not_exists(
   state:       'Channel::Filter::ReplyToBasedSender',
   frontend:    false
 )
+Setting.create_if_not_exists(
+  title:       'Define postmaster filter.',
+  name:        '0018_postmaster_import_archive',
+  area:        'Postmaster::PreFilter',
+  description: 'Define postmaster filter to import archive mailboxes.',
+  options:     {},
+  state:       'Channel::Filter::ImportArchive',
+  frontend:    false
+)
 Setting.create_if_not_exists(
   title:       'Defines postmaster filter.',
   name:        '0012_postmaster_filter_sender_is_system_address',

lib/email_helper/probe.rb

@@ -93,6 +93,8 @@ returns on fail
           return {
             result:             'ok',
             content_messages:   result_inbound[:content_messages],
+            archive_possible:   result_inbound[:archive_possible],
+            archive_week_range: result_inbound[:archive_week_range],
             setting:            settings,
           }
         end
@@ -125,6 +127,8 @@ returns on fail
         success                     = true
         result[:setting][:inbound]  = config
         result[:content_messages]   = result_inbound[:content_messages]
+        result[:archive_possible]   = result_inbound[:archive_possible]
+        result[:archive_week_range] = result_inbound[:archive_week_range]
 
         break
       end

spec/models/channel/email_parser_spec.rb

@@ -1242,7 +1242,7 @@ RSpec.describe Channel::EmailParser, type: :model do
       it 'applies the OutOfOfficeCheck filter to given message' do
         expect(Channel::Filter::OutOfOfficeCheck)
           .to receive(:run)
-          .with(kind_of(Hash), hash_including(subject: subject_line))
+          .with(kind_of(Hash), hash_including(subject: subject_line), kind_of(Hash))
 
         described_class.new.process({}, raw_mail)
       end

spec/models/channel/filter/import_archive_spec.rb

@@ -0,0 +1,247 @@
+require 'rails_helper'
+
+RSpec.describe Channel::Filter::ImportArchive do
+
+  let!(:agent1) { create(:agent, groups: Group.all) }
+
+  let(:channel_as_model) do
+    Channel.new(options: { inbound: { options: { archive: true } } })
+  end
+
+  let(:channel_as_hash) do
+    { options: { inbound: { options: { archive: true } } } }
+  end
+
+  let(:mail001) do
+    email_file_path = Rails.root.join('test/data/mail/mail001.box')
+    File.read(email_file_path)
+  end
+
+  let(:email_parse_mail001) do
+    email_raw_string = mail001
+    Channel::EmailParser.new.process(channel_as_model, email_raw_string)
+  end
+
+  let(:email_parse_mail001_hash) do
+    email_raw_string = mail001
+    Channel::EmailParser.new.process(channel_as_hash, email_raw_string)
+  end
+
+  let(:email_parse_mail001_answer) do
+    email_raw_string = mail001
+    email_raw_string.gsub!('Date: Thu, 3 May 2012 11:36:43 +0200', 'Date: Thu, 3 May 2014 11:36:43 +0200')
+    email_raw_string.gsub!('Message-Id: <053EA3703574649ABDAF24D43A05604F327A130@MEMASFRK004.example.com>', "In-Reply-To: <053EA3703574649ABDAF24D43A05604F327A130@MEMASFRK004.example.com>\nMessage-Id: <053EA3703574649ABDAF24D43A05604F327A130-1@MEMASFRK004.example.com>")
+
+    Channel::EmailParser.new.process(channel_as_model, email_raw_string)
+  end
+
+  shared_examples 'import archive base checks' do |ticket_create_date, article_create_date, article_count|
+    it 'checks if the state is closed' do
+      ticket1_p, _article1_p, _user1_p = email_parse_mail001
+      expect(ticket1_p.state.name).to eq('closed')
+    end
+
+    it 'checks if the article got created' do
+      ticket1_p, _article1_p, _user1_p = email_parse_mail001
+      expect(ticket1_p.articles.count).to eq(article_count)
+    end
+
+    it 'checks if the ticket create date is correct' do
+      ticket1_p, _article1_p, _user1_p = email_parse_mail001
+      expect(ticket1_p.created_at).to eq(Time.zone.parse(ticket_create_date))
+    end
+
+    it 'checks if the article create date is correct' do
+      _ticket1_p, article1_p, _user1_p = email_parse_mail001
+      expect(article1_p.created_at).to eq(Time.zone.parse(article_create_date))
+    end
+  end
+
+  shared_examples 'import archive answer checks' do |ticket_create_date, article_create_date, article_count|
+    it 'checks if the state is closed' do
+      ticket1_p, _article1_p, _user1_p = email_parse_mail001_answer
+      expect(ticket1_p.state.name).to eq('closed')
+    end
+
+    it 'checks if the article got created' do
+      ticket1_p, _article1_p, _user1_p = email_parse_mail001_answer
+      expect(ticket1_p.articles.count).to eq(article_count)
+    end
+
+    it 'checks if the ticket create date is correct' do
+      ticket1_p, _article1_p, _user1_p = email_parse_mail001_answer
+      expect(ticket1_p.created_at).to eq(Time.zone.parse(ticket_create_date))
+    end
+
+    it 'checks if the article create date is correct' do
+      _ticket1_p, article1_p, _user1_p = email_parse_mail001_answer
+      expect(article1_p.created_at).to eq(Time.zone.parse(article_create_date))
+    end
+  end
+
+  shared_examples 'notification sent checks' do |notification_count, parse_hash = false|
+    def email_hash(parse_hash)
+      if parse_hash
+        email_parse_mail001_hash
+      else
+        email_parse_mail001
+      end
+    end
+
+    before do
+      ticket1_p, article1_p, _user1_p = email_hash(parse_hash)
+
+      Scheduler.worker(true)
+      ticket1_p.reload
+      article1_p.reload
+    end
+
+    it 'verifies if notifications are sent' do
+      ticket1_p, _article1_p, _user1_p = email_hash(parse_hash)
+      expect(NotificationFactory::Mailer.already_sent?(ticket1_p, agent1, 'email')).to eq(notification_count)
+    end
+  end
+
+  describe '.run' do
+
+    context 'when initial ticket (import before outdated)' do
+      let(:channel_as_model) do
+        Channel.new(options: { inbound: { options: { archive: true, archive_before: '2012-03-04 00:00:00' } } })
+      end
+
+      include_examples 'notification sent checks', 1
+    end
+
+    context 'when initial ticket (import before matched)' do
+      let(:channel_as_model) do
+        Channel.new(options: { inbound: { options: { archive: true, archive_before: '2012-05-04 00:00:00' } } })
+      end
+
+      include_examples 'notification sent checks', 0
+    end
+
+    context 'when initial ticket (import till outdated)' do
+      let(:channel_as_model) do
+        Channel.new(options: { inbound: { options: { archive: true, archive_till: (Time.zone.now - 1.day).to_s } } })
+      end
+
+      include_examples 'notification sent checks', 1
+    end
+
+    context 'when initial ticket (import till matched)' do
+      let(:channel_as_model) do
+        Channel.new(options: { inbound: { options: { archive: true, archive_till: (Time.zone.now + 1.day).to_s } } })
+      end
+
+      include_examples 'notification sent checks', 0
+    end
+
+    context 'when initial ticket (import before outdated) with channel hash' do
+      let(:channel_as_hash) do
+        { options: { inbound: { options: { archive: true, archive_before: '2012-03-04 00:00:00' } } } }
+      end
+
+      include_examples 'notification sent checks', 1, true
+    end
+
+    context 'when initial ticket (import before matched) with channel hash' do
+      let(:channel_as_hash) do
+        { options: { inbound: { options: { archive: true, archive_before: '2012-05-04 00:00:00' } } } }
+      end
+
+      include_examples 'notification sent checks', 0, true
+    end
+
+    context 'when initial ticket (import till outdated) with channel hash' do
+      let(:channel_as_hash) do
+        { options: { inbound: { options: { archive: true, archive_till: (Time.zone.now - 1.day).to_s } } } }
+      end
+
+      include_examples 'notification sent checks', 1, true
+    end
+
+    context 'when initial ticket (import till matched) with channel hash' do
+      let(:channel_as_hash) do
+        { options: { inbound: { options: { archive: true, archive_till: (Time.zone.now + 1.day).to_s } } } }
+      end
+
+      include_examples 'notification sent checks', 0, true
+    end
+
+    context 'when initial ticket' do
+
+      include_examples 'import archive base checks', 'Thu, 03 May 2012 09:36:43 UTC +00:00', 'Thu, 03 May 2012 09:36:43 UTC +00:00', 1
+
+      context 'with scheduler run' do
+        before do
+          ticket1_p, article1_p, _user1_p = email_parse_mail001
+          Scheduler.worker(true)
+          ticket1_p.reload
+          article1_p.reload
+        end
+
+        include_examples 'import archive base checks', 'Thu, 03 May 2012 09:36:43 UTC +00:00', 'Thu, 03 May 2012 09:36:43 UTC +00:00', 1
+
+        it 'verifies if notifications are sent' do
+          ticket1_p, _article1_p, _user1_p = email_parse_mail001
+          expect(NotificationFactory::Mailer.already_sent?(ticket1_p, agent1, 'email')).to eq(0)
+        end
+
+        context 'when follow up check (mail answer)' do
+
+          include_examples 'import archive answer checks', 'Thu, 03 May 2012 09:36:43 UTC +00:00', 'Thu, 03 May 2014 09:36:43 UTC +00:00', 2
+
+          it 'checks if the article is different to the first one' do
+            _ticket1_p, article1_p, _user1_p = email_parse_mail001
+            _ticket2_p, article2_p, _user2_p = email_parse_mail001_answer
+            expect(article2_p.id).not_to eq(article1_p.id)
+          end
+
+          it 'checks if the article is a followup for the existing ticket' do
+            ticket1_p, _article1_p, _user1_p = email_parse_mail001
+            ticket2_p, _article2_p, _user2_p = email_parse_mail001_answer
+            expect(ticket2_p.id).to eq(ticket1_p.id)
+          end
+
+          context 'with scheduler run' do
+            before do
+              ticket2_p, article2_p, _user2_p = email_parse_mail001_answer
+              Scheduler.worker(true)
+              ticket2_p.reload
+              article2_p.reload
+            end
+
+            include_examples 'import archive answer checks', 'Thu, 03 May 2012 09:36:43 UTC +00:00', 'Thu, 03 May 2014 09:36:43 UTC +00:00', 2
+
+            it 'verifies if notifications are sent' do
+              ticket2_p, _article2_p, _user2_p = email_parse_mail001_answer
+              expect(NotificationFactory::Mailer.already_sent?(ticket2_p, agent1, 'email')).to eq(0)
+            end
+          end
+        end
+      end
+    end
+
+    context 'when duplicate check with channel as model' do
+      before do
+        Channel::EmailParser.new.process(channel_as_model, mail001)
+      end
+
+      it 'checks that the ticket count does not change on duplicates' do
+        expect { Channel::EmailParser.new.process(channel_as_model, mail001) }
+          .not_to change(Ticket, :count)
+      end
+    end
+
+    context 'when duplicate check with channel as hash' do
+      before do
+        Channel::EmailParser.new.process(channel_as_hash, mail001)
+      end
+
+      it 'checks that the ticket count does not change on duplicates' do
+        expect { Channel::EmailParser.new.process(channel_as_hash, mail001) }
+          .not_to change(Ticket, :count)
+      end
+    end
+  end
+end

spec/models/channel/filter/out_of_office_check_spec.rb

@@ -1,6 +1,6 @@
 require 'rails_helper'
 
-RSpec.describe Channel::Filter::OutOfOfficeCheck do
+RSpec.describe Channel::Filter::OutOfOfficeCheck, type: :channel_filter do
   describe '.run' do
     let(:mail_hash) { Channel::EmailParser.new.parse(<<~RAW.chomp) }
       From: me@example.com
@@ -15,14 +15,14 @@ RSpec.describe Channel::Filter::OutOfOfficeCheck do
 
     shared_examples 'regular message' do
       it 'sets x-zammad-out-of-office header to false' do
-        expect { described_class.run({}, mail_hash) }
+        expect { filter(mail_hash) }
           .to change { mail_hash[:'x-zammad-out-of-office'] }.to(false)
       end
     end
 
     shared_examples 'auto-response' do
       it 'sets x-zammad-out-of-office header to true' do
-        expect { described_class.run({}, mail_hash) }
+        expect { filter(mail_hash) }
           .to change { mail_hash[:'x-zammad-out-of-office'] }.to(true)
       end
     end

spec/support/channel_filter.rb

@@ -10,8 +10,8 @@ module ChannelFilterHelper
   #  filter({:'x-zammad-ticket-id' => 1234, ...})
   #
   # @return [nil]
-  def filter(mail_hash, channel: {})
+  def filter(mail_hash, channel: {}, transaction_params: {})
-    described_class.run(channel, mail_hash)
+    described_class.run(channel, mail_hash, transaction_params)
   end
 
   # Provides a helper method to parse a mail String and run the current class Channel::Filter.