Fixed issue #2200 - Unable to process email EncodingError: could not find a valid input encoding. Needed to downgrade mail gem because of parsing issues of valid mime issues.

Gemfile

@@ -80,7 +80,7 @@ gem 'twitter'
 
 # channels - email additions
 gem 'htmlentities'
-gem 'mail', '>= 2.7.1.rc1'
+gem 'mail', '2.6.6'
 gem 'mime-types'
 gem 'rchardet', '>= 1.8.0'
 gem 'valid_email2'

Gemfile.lock

@@ -220,15 +220,14 @@ GEM
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     lumberjack (1.0.12)
-    mail (2.7.1.rc1)
+    mail (2.6.6)
-      mini_mime (>= 0.1.1)
+      mime-types (>= 1.16, < 4)
     memoizable (0.4.2)
       thread_safe (~> 0.3, >= 0.3.1)
     method_source (0.9.0)
     mime-types (3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2016.0521)
-    mini_mime (1.0.0)
     mini_portile2 (2.3.0)
     minitest (5.11.3)
     multi_json (1.12.2)
@@ -519,7 +518,7 @@ DEPENDENCIES
   json
   koala
   libv8
-  mail (>= 2.7.1.rc1)
+  mail (= 2.6.6)
   mime-types
   mysql2
   net-ldap

app/models/channel/email_parser.rb

@@ -70,15 +70,16 @@ class Channel::EmailParser
 =end
 
   def parse(msg)
-    mail = Mail.new(msg.utf8_encode(from: charset_from_headers_of(msg)))
+    mail = Mail.new(msg.force_encoding('binary'))
 
+    headers = message_header_hash(mail)
+    body = message_body_hash(mail)
     message_attributes = [
       { mail_instance: mail },
-      message_header_hash(mail),
+      headers,
-      message_body_hash(mail),
+      body,
-      self.class.sender_attributes(mail),
+      self.class.sender_attributes(headers),
     ]
-
     message_attributes.reduce({}.with_indifferent_access, &:merge)
   end
 
@@ -340,8 +341,8 @@ returns
   end
 
   def self.sender_attributes(from)
-    if from.is_a?(Mail::Message)
+    if from.is_a?(HashWithIndifferentAccess)
-      from = SENDER_FIELDS.map { |f| from.header[f] }.compact
+      from = SENDER_FIELDS.map { |f| from[f] }.compact
                           .map(&:to_utf8).reject(&:blank?)
                           .partition { |address| address.match?(EMAIL_REGEX) }
                           .flatten.first
@@ -474,21 +475,16 @@ process unprocessable_mails (tmp/unprocessable_mail/*.eml) again
 
   private
 
-  def charset_from_headers_of(msg)
-    Mail.new(msg.b)
-        .header['Content-Type']
-        .try(:parameters)
-        .try(:[], :charset)
-  end
-
   def message_header_hash(mail)
     imported_fields = mail.header.fields.map do |f|
-      value = begin
+      begin
-                f.to_utf8
+        value = f.to_utf8
-              rescue NameError # handle bug #1238 in Mail 2.7.1.rc1
+        if value.blank?
-                ''             # swap out for commented line below once upgrade is available
+          value = f.raw_value.to_utf8
+        end
+      rescue
+        value = f.raw_value.to_utf8(fallback: :read_as_sanitized_binary)
       end
-
       [f.name.downcase, value]
     end.to_h
 
@@ -577,7 +573,7 @@ process unprocessable_mails (tmp/unprocessable_mail/*.eml) again
 
   def get_attachments(file, attachments, mail)
     return file.parts.map { |p| get_attachments(p, attachments, mail) } if file.parts.any?
-    return [] if [mail.text_part, mail.html_part].include?(file)
+    return [] if [mail.text_part&.body&.encoded, mail.html_part&.body&.encoded].include?(file.body.encoded)
 
     # get file preferences
     headers_store = {}
@@ -750,7 +746,11 @@ module Mail
   # workaround to get content of no parseable headers - in most cases with non 7 bit ascii signs
   class Field
     def raw_value
+      begin
         value = @raw_value.try(:utf8_encode)
+      rescue
+        value = @raw_value.utf8_encode(fallback: :read_as_sanitized_binary)
+      end
       return value if value.blank?
       value.sub(/^.+?:(\s|)/, '')
     end
@@ -791,4 +791,19 @@ module Mail
       end.join('')
     end
   end
+
+  # issue#348 - IMAP mail fetching stops because of broken spam email (e. g. broken Content-Transfer-Encoding value see test/fixtures/mail43.box)
+  # https://github.com/zammad/zammad/issues/348
+  class Body
+    def decoded
+      if !Encodings.defined?(encoding)
+        #raise UnknownEncodingType, "Don't know how to decode #{encoding}, please call #encoded and decode it yourself."
+        Rails.logger.info "UnknownEncodingType: Don't know how to decode #{encoding}!"
+        raw_source
+      else
+        Encodings.get_encoding(encoding).decode(raw_source)
+      end
+    end
+  end
+
 end

test/data/mail/mail018.yml

@@ -2,8 +2,8 @@
 from: postmaster@example.com
 from_email: postmaster@example.com
 from_display_name: ''
-subject: "Benachrichtung  \tzum  \t=?unicode-1-1-utf-7?Q?+ANw-bermittlungsstatus  \t(Fehlgeschlagen)?="
 to: sales@znuny.org
+subject: Benachrichtung zum =?unicode-1-1-utf-7?Q?+ANw-bermittlungsstatus (Fehlgeschlagen)?=
 body: |+
   Dies ist eine automatisch erstellte Benachrichtigung +APw-ber den Zustellstatus.
 

test/data/mail/mail046.yml

@@ -1,13 +1,18 @@
 --- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+from: '"ÎäŔźłÉ" <Glopelf7121@example.com>'
+from_email: Glopelf7121@example.com
+from_display_name: ÎäŔźłÉ
+to: info@example.de
+subject: '转发：整体提升企业服务水平'
 body: "Software zur Erkennung von \"Spam\" auf dem Rechner\n\n    bob.example.io\n\nhat
-  die eingegangene E-mail als m鰃liche \"Spam\"-Nachricht identifiziert.\nDie urspr黱gliche
+  die eingegangene E-mail als mögliche \"Spam\"-Nachricht identifiziert.\nDie ursprüngliche
-  Nachricht wurde an diesen Bericht angeh鋘gt, so dass\nSie sie anschauen k鰊nen (falls
+  Nachricht wurde an diesen Bericht angehängt, so dass\nSie sie anschauen können (falls
-  es doch eine legitime E-Mail ist) oder\n鋒nliche unerw黱schte Nachrichten in Zukunft
+  es doch eine legitime E-Mail ist) oder\nähnliche unerwünschte Nachrichten in Zukunft
-  markieren k鰊nen.\nBei Fragen zu diesem Vorgang wenden Sie sich bitte an\n\n    the
+  markieren können.\nBei Fragen zu diesem Vorgang wenden Sie sich bitte an\n\n    the
-  administrator of that system\n\nVorschau: “Ladies and gentlemen,” he said loudly,
+  administrator of that system\n\nVorschau: ¡°Ladies and gentlemen,¡± he said loudly,
-  waving for quiet.\n   “What an extraordinary moment this is! The perfect moment
+  waving for quiet.\n   ¡°What an extraordinary moment this is! The perfect moment
   for me to make\n   a little announcement I've been sitting on for some time! [...]
-  \n\nInhaltsanalyse im Detail:   (9.4 Punkte, 5.0 ben鰐igt)\n\nPkte Regelname              Beschreibung\n----
+  \n\nInhaltsanalyse im Detail:   (9.4 Punkte, 5.0 benötigt)\n\nPkte Regelname              Beschreibung\n----
   ---------------------- --------------------------------------------------\n 0.0
   RCVD_IN_DNSWL_BLOCKED  RBL: ADMINISTRATOR NOTICE: The query to DNSWL\n                            was
   blocked.  See\n                            http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block\n
@@ -16,14 +21,14 @@ body: "Software zur Erkennung von \"Spam\" auf dem Rechner\n\n    bob.example.io
   \                           [127.0.0.1 listed in bb.barracudacentral.org]\n 1.0
   RCVD_IN_PBL            RBL: Received via a relay in Spamhaus PBL\n                            [127.0.0.1
   listed in zen.spamhaus.org]\n 0.0 MIME_HTML_MOSTLY       BODY: Mehrteilige MIME-Nachricht
-  黚erwiegend in HTML\n 0.0 HTML_MESSAGE           BODY: Nachricht enth鋖t HTML\n 0.7
+  überwiegend in HTML\n 0.0 HTML_MESSAGE           BODY: Nachricht enthält HTML\n
-  MPART_ALT_DIFF         BODY: Nachrichtentext im Text- und HTML-Format\n                            unterscheiden
+  0.7 MPART_ALT_DIFF         BODY: Nachrichtentext im Text- und HTML-Format\n                            unterscheiden
   sich\n 1.6 RDNS_NONE              Delivered to internal network by a host with no
-  rDNS\n 1.1 SUBJ_ILLEGAL_CHARS     Betreff enth鋖t zu viele ung黮tige Zeichen\n 0.1
+  rDNS\n 1.1 SUBJ_ILLEGAL_CHARS     Betreff enthält zu viele ungültige Zeichen\n 0.1
   SUBJECT_NEEDS_ENCODING Subject is encoded but does not specify the\n                            encoding\n
-  3.2 HELO_DYNAMIC_IPADDR    HELO-Rechnername verd鋍htig (IP-Adresse 1)\n\nDie urspr黱gliche
+  3.2 HELO_DYNAMIC_IPADDR    HELO-Rechnername verdächtig (IP-Adresse 1)\n\nDie ursprüngliche
-  Nachricht enthielt nicht ausschlie遧ich Klartext\n(plain text) und kann eventuell
+  Nachricht enthielt nicht ausschließlich Klartext\n(plain text) und kann eventuell
-  eine Gefahr f黵 einige E-Mail-Programme\ndarstellen (falls sie z.B. einen Computervirus
+  eine Gefahr für einige E-Mail-Programme\ndarstellen (falls sie z.B. einen Computervirus
-  enth鋖t).\nM鯿hten Sie die Nachricht dennoch ansehen, ist es wahrscheinlich\nsicherer,
+  enthält).\nMöchten Sie die Nachricht dennoch ansehen, ist es wahrscheinlich\nsicherer,
   sie zuerst in einer Datei zu speichern und diese Datei danach\nmit einem Texteditor
-  zu 鰂fnen.\n\n"
+  zu öffnen.\n\n"

test/data/mail/mail069.yml

@@ -0,0 +1,31 @@
+--- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+from: Online-apotheke <ahromdn@example.com>
+from_email: ahromdn@example.com
+from_display_name: Online-apotheke
+to: kontakt@example.de
+subject: Online-apotheke. Günstigster Preis. Ohne Rezepte
+body: "Software zur Erkennung von \"Spam\" auf dem Rechner\n\n    hedwig.example.io\n\nhat
+  die eingegangene E-mail als mögliche \"Spam\"-Nachricht identifiziert.\nDie ursprüngliche
+  Nachricht wurde an diesen Bericht angehängt, so dass\nSie sie anschauen können (falls
+  es doch eine legitime E-Mail ist) oder\nähnliche unerwünschte Nachrichten in Zukunft
+  markieren können.\nBei Fragen zu diesem Vorgang wenden Sie sich bitte an\n\n    the
+  administrator of that system\n\nVorschau: [...] \n\nInhaltsanalyse im Detail:   (6.4
+  Punkte, 5.0 benötigt)\n\nPkte Regelname              Beschreibung\n---- ----------------------
+  --------------------------------------------------\n 2.5 URIBL_DBL_SPAM         Contains
+  a spam URL listed in the Spamhaus DBL\n                            blocklist\n                            [URIs:
+  chromis.group]\n 1.7 URIBL_BLACK            Contains an URL listed in the URIBL
+  blacklist\n                            [URIs: chromis.group]\n 1.9 URIBL_ABUSE_SURBL
+  \     Enthält URL in ABUSE-Liste (www.surbl.org) -\n                            changed
+  from JP to ABUSE bug 7279\n                            [URIs: chromis.group]\n-0.0
+  SPF_PASS               SPF: Senderechner entspricht SPF-Datensatz\n 0.7 MPART_ALT_DIFF
+  \        BODY: Nachrichtentext im Text- und HTML-Format\n                            unterscheiden
+  sich\n 0.0 HTML_MESSAGE           BODY: Nachricht enthält HTML\n 0.3 HTML_IMAGE_ONLY_04
+  \    BODY: Außer Bildern nur 0-400 Zeichen Text\n 0.1 HTML_SHORT_LINK_IMG_1  HTML
+  is very short with a linked image\n-1.0 MAILING_LIST_MULTI     Multiple indicators
+  imply a widely-seen list\n                            manager\n\nDie ursprüngliche
+  Nachricht enthielt nicht ausschließlich Klartext\n(plain text) und kann eventuell
+  eine Gefahr für einige E-Mail-Programme\ndarstellen (falls sie z.B. einen Computervirus
+  enthält).\nMöchten Sie die Nachricht dennoch ansehen, ist es wahrscheinlich\nsicherer,
+  sie zuerst in einer Datei zu speichern und diese Datei danach\nmit einem Texteditor
+  zu öffnen.\n\n"
+content_type: text/plain

test/unit/email_build_test.rb

@@ -56,15 +56,14 @@ class EmailBuildTest < ActiveSupport::TestCase
       ],
     )
 
-    text_should = Mail::Utilities.to_crlf(<<~MSG_TEXT.chomp)
+    text_should = <<~MSG_TEXT.chomp
       > Welcome!
       >
       > Thank you for installing Zammad. äöüß
       >
       MSG_TEXT
-    html_should = Mail::Utilities.to_crlf(html)
     assert_equal(text_should, mail.text_part.body.to_s)
-    assert_equal(html_should, mail.html_part.body.to_s)
+    assert_equal(html, mail.html_part.body.to_s)
 
     parser = Channel::EmailParser.new
     data = parser.parse(mail.to_s)
@@ -115,9 +114,7 @@ class EmailBuildTest < ActiveSupport::TestCase
       ],
     )
 
-    mail_gem_should = Mail::Utilities.to_crlf(text)
+    assert_equal(text, mail.text_part.body.to_s)
-    email_parser_should = text
-    assert_equal(mail_gem_should, mail.text_part.body.to_s)
     assert_nil(mail.html_part)
     assert_equal('image/png; filename=somename.png', mail.attachments[0].content_type)
 
@@ -125,7 +122,7 @@ class EmailBuildTest < ActiveSupport::TestCase
     data = parser.parse(mail.to_s)
 
     # check body
-    assert_equal(email_parser_should, data[:body])
+    assert_equal(text, data[:body])
 
     # check count of attachments, 2
     assert_equal(1, data[:attachments].length)
@@ -197,9 +194,7 @@ class EmailBuildTest < ActiveSupport::TestCase
       ],
     )
 
-    mail_gem_should = Mail::Utilities.to_crlf(text)
+    assert_equal(text, mail.text_part.body.to_s)
-    email_parser_should = text
-    assert_equal(mail_gem_should, mail.text_part.body.to_s)
     assert_nil(mail.html_part)
     assert_equal('text/calendar; filename=schedule.ics', mail.attachments[0].content_type)
 
@@ -207,7 +202,7 @@ class EmailBuildTest < ActiveSupport::TestCase
     data = parser.parse(mail.to_s)
 
     # check body
-    assert_equal(email_parser_should, data[:body])
+    assert_equal(text, data[:body])
 
     # check count of attachments, 2
     assert_equal(1, data[:attachments].length)
@@ -238,16 +233,14 @@ class EmailBuildTest < ActiveSupport::TestCase
       body: text,
     )
 
-    mail_gem_should = Mail::Utilities.to_crlf(text)
+    assert_equal(text, mail.body.to_s)
-    email_parser_should = text
-    assert_equal(mail_gem_should, mail.body.to_s)
     assert_nil(mail.html_part)
 
     parser = Channel::EmailParser.new
     data = parser.parse(mail.to_s)
 
     # check body
-    assert_equal(email_parser_should, data[:body])
+    assert_equal(text, data[:body])
 
     # check count of attachments, 0
     assert_equal(0, data[:attachments].length)

test/unit/email_parser_test.rb

@@ -3,6 +3,15 @@
 require 'test_helper'
 
 class EmailParserTest < ActiveSupport::TestCase
+
+=begin
+
+to write new .yml files for emails you can use the following code:
+
+File.write('test/data/mail/mailXXX.yml', Channel::EmailParser.new.parse(File.read('test/data/mail/mailXXX.box')).slice(:from, :from_email, :from_display_name, :to, :cc, :subject, :body, :content_type, :'reply-to').to_yaml)
+
+=end
+
   test 'parse' do
     msg_files = Dir.glob(Rails.root.join('test', 'data', 'mail', 'mail*.box')).sort
 
@@ -19,15 +28,10 @@ class EmailParserTest < ActiveSupport::TestCase
       # assert: raw content hash is a subset of parsed message hash
       expected_msg = m[:content].except(:attachments)
       parsed_msg = m[:parsed].slice(*expected_msg.keys)
-      failure_msg = [parsed_msg, expected_msg]
-                      .map(&:to_a).map(&:sort).reduce(&:zip)
-                      .reject { |a| a.uniq.one? }
-                      .map { |a, b| "#{a.first.upcase}\n  #{m[:source]}: #{a.last}\n  #{m[:source].ext('yml')}: #{b.last}" }
-                      .join("\n")
 
-      assert_operator(expected_msg, :<=, parsed_msg,
+      expected_msg.each do |key, value|
-                      "parsed message data does not match message content:\n" +
+        assert_equal(value, parsed_msg[key], "parsed message data does not match test/data/mail/#{m[:source]}: #{key}")
-                      failure_msg)
+      end
 
       # assert: attachments in parsed message hash match metadata in raw hash
       next if m[:content][:attachments].blank?

test/unit/email_process_test.rb

@@ -48,7 +48,7 @@ Some Text",
       {
         data: "From: me@example.com
 To: customer@example.com
-Subject: äöü some subject
+Subject: äöü some subject 1
 
 Some Textäöü",
         channel: {
@@ -58,7 +58,7 @@ Some Textäöü",
         result: {
           0 => {
             priority: '2 normal',
-            title: 'äöü some subject',
+            title: 'äöü some subject 1',
           },
           1 => {
             body: 'Some Textäöü',
@@ -71,7 +71,7 @@ Some Textäöü",
       {
         data: "From: me@exampl'e.com
 To: customer@exampl'e.com
-Subject: äöü some subject
+Subject: äöü some subject 2
 
 Some Textäöü",
         channel: {
@@ -81,7 +81,7 @@ Some Textäöü",
         result: {
           0 => {
             priority: '2 normal',
-            title: 'äöü some subject',
+            title: 'äöü some subject 2',
           },
           1 => {
             body: 'Some Textäöü',
@@ -155,17 +155,17 @@ Some Textäöü without subject#2",
       {
         data: "From: me@example.com
 To: customer@example.com
-Subject: äöü some subject
+Subject: äöü some subject 3
 
 Some Textäöü".encode('ISO-8859-1'),
         success: true,
         result: {
           0 => {
             priority: '2 normal',
-            title: 'äöü some subject',
+            title: '??? some subject 3', # it's ok, because subject need to be 7bit encoded
           },
           1 => {
-            body: 'Some Textäöü',
+            body: 'Some Text???', # it's ok, because no content-type is given
             sender: 'Customer',
             type: 'email',
             internal: false,
@@ -627,7 +627,7 @@ Some Text",
         result: {
           0 => {
             priority: '2 normal',
-            title: 'Subject: 【专业为您注册香港及海外公司（好处多多）】　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　',
+            title: '【专业为您注册香港及海外公司（好处多多）】　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　',
           },
           1 => {
             body: 'Some Text',
@@ -2758,7 +2758,7 @@ Some Text',
             title: '转发：整体提升企业服务水平',
           },
           1 => {
-            from: '"武兰成" <Glopelf7121@example.com>',
+            from: '"ÎäŔźłÉ" <Glopelf7121@example.com>',
             sender: 'Customer',
             type: 'email',
           },
@@ -2766,9 +2766,9 @@ Some Text',
         verify: {
           users: [
             {
-              firstname: '武兰成',
+              firstname: 'ÎäŔźłÉ',
               lastname: '',
-              fullname: '武兰成',
+              fullname: 'ÎäŔźłÉ',
               email: 'glopelf7121@example.com',
             },
           ],
@@ -2990,7 +2990,7 @@ Some Text',
             title: 'some subject 3',
           },
           1 => {
-            from: '"Vandromme, Frédéric" <fvandromme@example.com>',
+            from: '=?windows-1258?B?VmFuZHJvbW1lLCBGculk6XJpYw==?= <fvandromme@example.com>',
             sender: 'Customer',
             type: 'email',
             body: 'Some Text',
@@ -3008,18 +3008,16 @@ Some Text',
         },
       },
       {
-        data: <<~RAW_MAIL.chomp,
+        data: 'From: me@example.com
-          From: me@example.com
+To: customer@example.com
-          To: customer@example.com
+Subject: some subject
-          Subject: some subject
+Content-Type: text/html; charset=us-ascii; format=flowed
-          Content-Type: text/html; charset=us-ascii; format=flowed
 
-          <html>
+<html>
   <body>
     <a href="mailto:testäöü@example.com">test</a>
   </body>
-          </html>
+</html>',
-          RAW_MAIL
         success: true,
         result: {
           0 => {

test/unit/ticket_test.rb

@@ -2,6 +2,11 @@
 require 'test_helper'
 
 class TicketTest < ActiveSupport::TestCase
+
+  setup do
+    Ticket.destroy_all
+  end
+
   test 'ticket create' do
     ticket = Ticket.create!(
       title: "some title\n äöüß",