Fixes #2063 - parses emails with empty msgid. Thanks to @dsukhin ❤️

app/models/channel/email_parser.rb

@@ -78,6 +78,8 @@ class Channel::EmailParser
     msg = Mail::Utilities.binary_unsafe_to_crlf(msg)
     mail = Mail.new(msg)
 
+    message_ensure_message_id(msg, mail)
+
     force_parts_encoding_if_needed(mail)
 
     headers = message_header_hash(mail)
@@ -545,6 +547,17 @@ process unprocessable_mails (tmp/unprocessable_mail/*.eml) again
     end
   end
 
+  # generate Message ID on the fly if it was missing
+  # yes, Mail gem generates one in some cases
+  # but it is 100% random so duplicate messages would not be detected
+  def message_ensure_message_id(raw, parsed)
+    field = parsed.header.fields.find { |elem| elem.name == 'Message-ID' }
+
+    return true if field&.unparsed_value.present?
+
+    parsed.message_id = generate_message_id(raw, parsed.from)
+  end
+
   def message_header_hash(mail)
     imported_fields = mail.header.fields.map do |f|
       begin
@@ -909,6 +922,18 @@ process unprocessable_mails (tmp/unprocessable_mail/*.eml) again
       'In-Reply-To': parsed_incoming_mail[:message_id],
     )
   end
+
+  def guess_email_fqdn(from)
+    Mail::Address.new(from).domain.strip
+  rescue
+    nil
+  end
+
+  def generate_message_id(raw_message, from)
+    fqdn = guess_email_fqdn(from) || 'zammad_generated'
+
+    "<gen-#{Digest::MD5.hexdigest(raw_message)}@#{fqdn}>"
+  end
 end
 
 module Mail

test/data/mail/mail095.box

@@ -0,0 +1,27 @@
+Return-Path: <user.name@example.com>
+Delivered-To: user@example.com
+Received: from host.example.com
+	by host.example.com with LMTP id kFSRC5u/I1tJHBAAz53O+w
+	for <user@example.com>; Fri, 15 Jun 2018 09:31:07 -0400
+Return-path: <user.name@example.com>
+Envelope-to: user@example.com
+Delivery-date: Fri, 15 Jun 2018 09:31:07 -0400
+Received: from mail-do1ram03ok0780.emails.users.example.com ([0.0.0.0]:15241 helo=do1ram03ok0780.emails.users.example.com)
+	by host.example.com with esmtps (TLSv1.2:ECDHE-RSA-AES256-SHA384:256)
+	(Exim 4.91)
+	(envelope-from <user.name@example.com>)
+	id 1dFool-1004vt-2z
+	for user@example.com; Fri, 15 Jun 2018 09:31:07 -0400
+From: "Name R. John" <user.name@example.com>
+To: To This Person <user@example.com>
+Subject: Subject of message
+Thread-Topic: Subject of message
+Thread-Index: AdQDSn2ZjObO2qLmRLWd8iIbgua/6gBYoT0g
+Sender: "Name R. John" <user.name@example.com>
+Date: Fri, 15 Jun 2018 13:30:23 +0000
+Message-ID:
+ <DM6PR02MF1425372C1E5187B91210554G127D5@do1ram03ok0780.emails.users.example.com>
+Accept-Language: en-US
+Content-Language: en-US
+
+Message text

test/data/mail/mail095.yml

@@ -0,0 +1,12 @@
+--- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+from: '"Name R. John" <user.name@example.com>'
+from_email: user.name@example.com
+from_display_name: Name R. John
+to: To This Person <user@example.com>
+subject: Subject of message
+body: 'Message text
+
+  '
+content_type: text/plain
+attachments: []
+message_id: "<DM6PR02MF1425372C1E5187B91210554G127D5@do1ram03ok0780.emails.users.example.com>"

test/data/mail/mail096.box

@@ -0,0 +1,26 @@
+Return-Path: <user.name@example.com>
+Delivered-To: user@example.com
+Received: from host.example.com
+	by host.example.com with LMTP id kFSRC5u/I1tJHBAAz53O+w
+	for <user@example.com>; Fri, 15 Jun 2018 09:31:07 -0400
+Return-path: <user.name@example.com>
+Envelope-to: user@example.com
+Delivery-date: Fri, 15 Jun 2018 09:31:07 -0400
+Received: from mail-do1ram03ok0780.emails.users.example.com ([0.0.0.0]:15241 helo=do1ram03ok0780.emails.users.example.com)
+	by host.example.com with esmtps (TLSv1.2:ECDHE-RSA-AES256-SHA384:256)
+	(Exim 4.91)
+	(envelope-from <user.name@example.com>)
+	id 1dFool-1004vt-2z
+	for user@example.com; Fri, 15 Jun 2018 09:31:07 -0400
+From: "Name R. John" <user.name@example.com>
+To: To This Person <user@example.com>
+Subject: Subject of message
+Thread-Topic: Subject of message
+Thread-Index: AdQDSn2ZjObO2qLmRLWd8iIbgua/6gBYoT0g
+Sender: "Name R. John" <user.name@example.com>
+Date: Fri, 15 Jun 2018 13:30:23 +0000
+Message-ID:
+Accept-Language: en-US
+Content-Language: en-US
+
+Message text

test/data/mail/mail096.yml

@@ -0,0 +1,12 @@
+--- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+from: '"Name R. John" <user.name@example.com>'
+from_email: user.name@example.com
+from_display_name: Name R. John
+to: To This Person <user@example.com>
+subject: Subject of message
+body: 'Message text
+
+  '
+content_type: text/plain
+attachments: []
+message_id: "<gen-baa1c9c250966ffb2bb20a9d611c2e5a@zammad_generated>"

test/data/mail/mail097.box

@@ -0,0 +1,25 @@
+Return-Path: <user.name@example.com>
+Delivered-To: user@example.com
+Received: from host.example.com
+	by host.example.com with LMTP id kFSRC5u/I1tJHBAAz53O+w
+	for <user@example.com>; Fri, 15 Jun 2018 09:31:07 -0400
+Return-path: <user.name@example.com>
+Envelope-to: user@example.com
+Delivery-date: Fri, 15 Jun 2018 09:31:07 -0400
+Received: from mail-do1ram03ok0780.emails.users.example.com ([0.0.0.0]:15241 helo=do1ram03ok0780.emails.users.example.com)
+	by host.example.com with esmtps (TLSv1.2:ECDHE-RSA-AES256-SHA384:256)
+	(Exim 4.91)
+	(envelope-from <user.name@example.com>)
+	id 1dFool-1004vt-2z
+	for user@example.com; Fri, 15 Jun 2018 09:31:07 -0400
+From: "Name R. John" <user.name@example.com>
+To: To This Person <user@example.com>
+Subject: Subject of message
+Thread-Topic: Subject of message
+Thread-Index: AdQDSn2ZjObO2qLmRLWd8iIbgua/6gBYoT0g
+Sender: "Name R. John" <user.name@example.com>
+Date: Fri, 15 Jun 2018 13:30:23 +0000
+Accept-Language: en-US
+Content-Language: en-US
+
+Message text

test/data/mail/mail097.yml

@@ -0,0 +1,12 @@
+--- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
+from: '"Name R. John" <user.name@example.com>'
+from_email: user.name@example.com
+from_display_name: Name R. John
+to: To This Person <user@example.com>
+subject: Subject of message
+body: 'Message text
+
+  '
+content_type: text/plain
+attachments: []
+message_id: "<gen-49df88528b60c554dbd3dcb4b067ecb2@zammad_generated>"