Sutty/trabajo-afectivo
5
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Fixed issue #2200 - Unable to process email EncodingError: could not find a valid input encoding. Needed to downgrade mail gem because of parsing issues of valid mime issues.

Browse source
This commit is contained in:
Martin Edenhofer 2018-08-22 12:05:14 +02:00
parent 35a2e991a6
commit 64c6d706c6
12 changed files with 2606 additions and 90 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Gemfile
View file

@ -80,7 +80,7 @@ gem 'twitter'
# channels - email additions
gem 'htmlentities'
gem 'mail', '>= 2.7.1.rc1'
gem 'mail', '2.6.6'
gem 'mime-types'
gem 'rchardet', '>= 1.8.0'
gem 'valid_email2'

7
Gemfile.lock
View file

@ -220,15 +220,14 @@ GEM
crass (~> 1.0.2)
nokogiri (>= 1.5.9)
lumberjack (1.0.12)
mail (2.7.1.rc1)
mini_mime (>= 0.1.1)
mail (2.6.6)
mime-types (>= 1.16, < 4)
memoizable (0.4.2)
thread_safe (~> 0.3, >= 0.3.1)
method_source (0.9.0)
mime-types (3.1)
mime-types-data (~> 3.2015)
mime-types-data (3.2016.0521)
mini_mime (1.0.0)
mini_portile2 (2.3.0)
minitest (5.11.3)
multi_json (1.12.2)
@ -519,7 +518,7 @@ DEPENDENCIES
json
koala
libv8
mail (>= 2.7.1.rc1)
mail (= 2.6.6)
mime-types
mysql2
net-ldap

55
app/models/channel/email_parser.rb
View file

@ -70,15 +70,16 @@ class Channel::EmailParser
=end
def parse(msg)
mail = Mail.new(msg.utf8_encode(from: charset_from_headers_of(msg)))
mail = Mail.new(msg.force_encoding('binary'))
headers = message_header_hash(mail)
body = message_body_hash(mail)
message_attributes = [
{ mail_instance: mail },
message_header_hash(mail),
message_body_hash(mail),
self.class.sender_attributes(mail),
headers,
body,
self.class.sender_attributes(headers),
]
message_attributes.reduce({}.with_indifferent_access, &:merge)
end
@ -340,8 +341,8 @@ returns
end
def self.sender_attributes(from)
if from.is_a?(Mail::Message)
from = SENDER_FIELDS.map { |f| from.header[f] }.compact
if from.is_a?(HashWithIndifferentAccess)
from = SENDER_FIELDS.map { |f| from[f] }.compact
.map(&:to_utf8).reject(&:blank?)
.partition { |address| address.match?(EMAIL_REGEX) }
.flatten.first
@ -474,21 +475,16 @@ process unprocessable_mails (tmp/unprocessable_mail/*.eml) again
private
def charset_from_headers_of(msg)
Mail.new(msg.b)
.header['Content-Type']
.try(:parameters)
.try(:[], :charset)
end
def message_header_hash(mail)
imported_fields = mail.header.fields.map do |f|
value = begin
f.to_utf8
rescue NameError # handle bug #1238 in Mail 2.7.1.rc1
'' # swap out for commented line below once upgrade is available
begin
value = f.to_utf8
if value.blank?
value = f.raw_value.to_utf8
end
rescue
value = f.raw_value.to_utf8(fallback: :read_as_sanitized_binary)
end
[f.name.downcase, value]
end.to_h
@ -577,7 +573,7 @@ process unprocessable_mails (tmp/unprocessable_mail/*.eml) again
def get_attachments(file, attachments, mail)
return file.parts.map { |p| get_attachments(p, attachments, mail) } if file.parts.any?
return [] if [mail.text_part, mail.html_part].include?(file)
return [] if [mail.text_part&.body&.encoded, mail.html_part&.body&.encoded].include?(file.body.encoded)
# get file preferences
headers_store = {}
@ -750,7 +746,11 @@ module Mail
# workaround to get content of no parseable headers - in most cases with non 7 bit ascii signs
class Field
def raw_value
begin
value = @raw_value.try(:utf8_encode)
rescue
value = @raw_value.utf8_encode(fallback: :read_as_sanitized_binary)
end
return value if value.blank?
value.sub(/^.+?:(\s|)/, '')
end
@ -791,4 +791,19 @@ module Mail
end.join('')
end
end
# issue#348 - IMAP mail fetching stops because of broken spam email (e. g. broken Content-Transfer-Encoding value see test/fixtures/mail43.box)
# https://github.com/zammad/zammad/issues/348
class Body
def decoded
if !Encodings.defined?(encoding)
#raise UnknownEncodingType, "Don't know how to decode #{encoding}, please call #encoded and decode it yourself."
Rails.logger.info "UnknownEncodingType: Don't know how to decode #{encoding}!"
raw_source
else
Encodings.get_encoding(encoding).decode(raw_source)
end
end
end
end

2
test/data/mail/mail018.yml
View file

@ -2,8 +2,8 @@
from: postmaster@example.com
from_email: postmaster@example.com
from_display_name: ''
subject: "Benachrichtung \tzum \t=?unicode-1-1-utf-7?Q?+ANw-bermittlungsstatus \t(Fehlgeschlagen)?="
to: sales@znuny.org
subject: Benachrichtung zum =?unicode-1-1-utf-7?Q?+ANw-bermittlungsstatus (Fehlgeschlagen)?=
body: |+
Dies ist eine automatisch erstellte Benachrichtigung +APw-ber den Zustellstatus.

35
test/data/mail/mail046.yml
View file

@ -1,13 +1,18 @@
--- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
from: '"ÎäŔźłÉ" <Glopelf7121@example.com>'
from_email: Glopelf7121@example.com
from_display_name: ÎäŔźłÉ
to: info@example.de
subject: '转发：整体提升企业服务水平'
body: "Software zur Erkennung von \"Spam\" auf dem Rechner\n\n bob.example.io\n\nhat
die eingegangene E-mail als m鰃liche \"Spam\"-Nachricht identifiziert.\nDie urspr黱gliche
Nachricht wurde an diesen Bericht angeh鋘gt, so dass\nSie sie anschauen k鰊nen (falls
es doch eine legitime E-Mail ist) oder\n鋒nliche unerw黱schte Nachrichten in Zukunft
markieren k鰊nen.\nBei Fragen zu diesem Vorgang wenden Sie sich bitte an\n\n the
administrator of that system\n\nVorschau: “Ladies and gentlemen,” he said loudly,
waving for quiet.\n “What an extraordinary moment this is! The perfect moment
die eingegangene E-mail als mögliche \"Spam\"-Nachricht identifiziert.\nDie ursprüngliche
Nachricht wurde an diesen Bericht angehängt, so dass\nSie sie anschauen können (falls
es doch eine legitime E-Mail ist) oder\nähnliche unerwünschte Nachrichten in Zukunft
markieren können.\nBei Fragen zu diesem Vorgang wenden Sie sich bitte an\n\n the
administrator of that system\n\nVorschau: ¡°Ladies and gentlemen,¡± he said loudly,
waving for quiet.\n ¡°What an extraordinary moment this is! The perfect moment
for me to make\n a little announcement I've been sitting on for some time! [...]
\n\nInhaltsanalyse im Detail: (9.4 Punkte, 5.0 ben鰐igt)\n\nPkte Regelname Beschreibung\n----
\n\nInhaltsanalyse im Detail: (9.4 Punkte, 5.0 benötigt)\n\nPkte Regelname Beschreibung\n----
---------------------- --------------------------------------------------\n 0.0
RCVD_IN_DNSWL_BLOCKED RBL: ADMINISTRATOR NOTICE: The query to DNSWL\n was
blocked. See\n http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block\n
@ -16,14 +21,14 @@ body: "Software zur Erkennung von \"Spam\" auf dem Rechner\n\n bob.example.io
\ [127.0.0.1 listed in bb.barracudacentral.org]\n 1.0
RCVD_IN_PBL RBL: Received via a relay in Spamhaus PBL\n [127.0.0.1
listed in zen.spamhaus.org]\n 0.0 MIME_HTML_MOSTLY BODY: Mehrteilige MIME-Nachricht
黚erwiegend in HTML\n 0.0 HTML_MESSAGE BODY: Nachricht enth鋖t HTML\n 0.7
MPART_ALT_DIFF BODY: Nachrichtentext im Text- und HTML-Format\n unterscheiden
überwiegend in HTML\n 0.0 HTML_MESSAGE BODY: Nachricht enthält HTML\n
0.7 MPART_ALT_DIFF BODY: Nachrichtentext im Text- und HTML-Format\n unterscheiden
sich\n 1.6 RDNS_NONE Delivered to internal network by a host with no
rDNS\n 1.1 SUBJ_ILLEGAL_CHARS Betreff enth鋖t zu viele ung黮tige Zeichen\n 0.1
rDNS\n 1.1 SUBJ_ILLEGAL_CHARS Betreff enthält zu viele ungültige Zeichen\n 0.1
SUBJECT_NEEDS_ENCODING Subject is encoded but does not specify the\n encoding\n
3.2 HELO_DYNAMIC_IPADDR HELO-Rechnername verd鋍htig (IP-Adresse 1)\n\nDie urspr黱gliche
Nachricht enthielt nicht ausschlie遧ich Klartext\n(plain text) und kann eventuell
eine Gefahr f黵 einige E-Mail-Programme\ndarstellen (falls sie z.B. einen Computervirus
enth鋖t).\nM鯿hten Sie die Nachricht dennoch ansehen, ist es wahrscheinlich\nsicherer,
3.2 HELO_DYNAMIC_IPADDR HELO-Rechnername verdächtig (IP-Adresse 1)\n\nDie ursprüngliche
Nachricht enthielt nicht ausschließlich Klartext\n(plain text) und kann eventuell
eine Gefahr für einige E-Mail-Programme\ndarstellen (falls sie z.B. einen Computervirus
enthält).\nMöchten Sie die Nachricht dennoch ansehen, ist es wahrscheinlich\nsicherer,
sie zuerst in einer Datei zu speichern und diese Datei danach\nmit einem Texteditor
zu 鰂fnen.\n\n"
zu öffnen.\n\n"

2466
test/data/mail/mail069.box Normal file
View file

File diff suppressed because it is too large Load diff

31
test/data/mail/mail069.yml Normal file
View file

@ -0,0 +1,31 @@
--- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
from: Online-apotheke <ahromdn@example.com>
from_email: ahromdn@example.com
from_display_name: Online-apotheke
to: kontakt@example.de
subject: Online-apotheke. Günstigster Preis. Ohne Rezepte
body: "Software zur Erkennung von \"Spam\" auf dem Rechner\n\n hedwig.example.io\n\nhat
die eingegangene E-mail als mögliche \"Spam\"-Nachricht identifiziert.\nDie ursprüngliche
Nachricht wurde an diesen Bericht angehängt, so dass\nSie sie anschauen können (falls
es doch eine legitime E-Mail ist) oder\nähnliche unerwünschte Nachrichten in Zukunft
markieren können.\nBei Fragen zu diesem Vorgang wenden Sie sich bitte an\n\n the
administrator of that system\n\nVorschau: [...] \n\nInhaltsanalyse im Detail: (6.4
Punkte, 5.0 benötigt)\n\nPkte Regelname Beschreibung\n---- ----------------------
--------------------------------------------------\n 2.5 URIBL_DBL_SPAM Contains
a spam URL listed in the Spamhaus DBL\n blocklist\n [URIs:
chromis.group]\n 1.7 URIBL_BLACK Contains an URL listed in the URIBL
blacklist\n [URIs: chromis.group]\n 1.9 URIBL_ABUSE_SURBL
\ Enthält URL in ABUSE-Liste (www.surbl.org) -\n changed
from JP to ABUSE bug 7279\n [URIs: chromis.group]\n-0.0
SPF_PASS SPF: Senderechner entspricht SPF-Datensatz\n 0.7 MPART_ALT_DIFF
\ BODY: Nachrichtentext im Text- und HTML-Format\n unterscheiden
sich\n 0.0 HTML_MESSAGE BODY: Nachricht enthält HTML\n 0.3 HTML_IMAGE_ONLY_04
\ BODY: Außer Bildern nur 0-400 Zeichen Text\n 0.1 HTML_SHORT_LINK_IMG_1 HTML
is very short with a linked image\n-1.0 MAILING_LIST_MULTI Multiple indicators
imply a widely-seen list\n manager\n\nDie ursprüngliche
Nachricht enthielt nicht ausschließlich Klartext\n(plain text) und kann eventuell
eine Gefahr für einige E-Mail-Programme\ndarstellen (falls sie z.B. einen Computervirus
enthält).\nMöchten Sie die Nachricht dennoch ansehen, ist es wahrscheinlich\nsicherer,
sie zuerst in einer Datei zu speichern und diese Datei danach\nmit einem Texteditor
zu öffnen.\n\n"
content_type: text/plain

23
test/unit/email_build_test.rb
View file

@ -56,15 +56,14 @@ class EmailBuildTest < ActiveSupport::TestCase
],
)
text_should = Mail::Utilities.to_crlf(<<~MSG_TEXT.chomp)
text_should = <<~MSG_TEXT.chomp
> Welcome!
>
> Thank you for installing Zammad. äöüß
>
MSG_TEXT
html_should = Mail::Utilities.to_crlf(html)
assert_equal(text_should, mail.text_part.body.to_s)
assert_equal(html_should, mail.html_part.body.to_s)
assert_equal(html, mail.html_part.body.to_s)
parser = Channel::EmailParser.new
data = parser.parse(mail.to_s)
@ -115,9 +114,7 @@ class EmailBuildTest < ActiveSupport::TestCase
],
)
mail_gem_should = Mail::Utilities.to_crlf(text)
email_parser_should = text
assert_equal(mail_gem_should, mail.text_part.body.to_s)
assert_equal(text, mail.text_part.body.to_s)
assert_nil(mail.html_part)
assert_equal('image/png; filename=somename.png', mail.attachments[0].content_type)
@ -125,7 +122,7 @@ class EmailBuildTest < ActiveSupport::TestCase
data = parser.parse(mail.to_s)
# check body
assert_equal(email_parser_should, data[:body])
assert_equal(text, data[:body])
# check count of attachments, 2
assert_equal(1, data[:attachments].length)
@ -197,9 +194,7 @@ class EmailBuildTest < ActiveSupport::TestCase
],
)
mail_gem_should = Mail::Utilities.to_crlf(text)
email_parser_should = text
assert_equal(mail_gem_should, mail.text_part.body.to_s)
assert_equal(text, mail.text_part.body.to_s)
assert_nil(mail.html_part)
assert_equal('text/calendar; filename=schedule.ics', mail.attachments[0].content_type)
@ -207,7 +202,7 @@ class EmailBuildTest < ActiveSupport::TestCase
data = parser.parse(mail.to_s)
# check body
assert_equal(email_parser_should, data[:body])
assert_equal(text, data[:body])
# check count of attachments, 2
assert_equal(1, data[:attachments].length)
@ -238,16 +233,14 @@ class EmailBuildTest < ActiveSupport::TestCase
body: text,
)
mail_gem_should = Mail::Utilities.to_crlf(text)
email_parser_should = text
assert_equal(mail_gem_should, mail.body.to_s)
assert_equal(text, mail.body.to_s)
assert_nil(mail.html_part)
parser = Channel::EmailParser.new
data = parser.parse(mail.to_s)
# check body
assert_equal(email_parser_should, data[:body])
assert_equal(text, data[:body])
# check count of attachments, 0
assert_equal(0, data[:attachments].length)

20
test/unit/email_parser_test.rb
View file

@ -3,6 +3,15 @@
require 'test_helper'
class EmailParserTest < ActiveSupport::TestCase
=begin
to write new .yml files for emails you can use the following code:
File.write('test/data/mail/mailXXX.yml', Channel::EmailParser.new.parse(File.read('test/data/mail/mailXXX.box')).slice(:from, :from_email, :from_display_name, :to, :cc, :subject, :body, :content_type, :'reply-to').to_yaml)
=end
test 'parse' do
msg_files = Dir.glob(Rails.root.join('test', 'data', 'mail', 'mail*.box')).sort
@ -19,15 +28,10 @@ class EmailParserTest < ActiveSupport::TestCase
# assert: raw content hash is a subset of parsed message hash
expected_msg = m[:content].except(:attachments)
parsed_msg = m[:parsed].slice(*expected_msg.keys)
failure_msg = [parsed_msg, expected_msg]
.map(&:to_a).map(&:sort).reduce(&:zip)
.reject { |a| a.uniq.one? }
.map { |a, b| "#{a.first.upcase}\n #{m[:source]}: #{a.last}\n #{m[:source].ext('yml')}: #{b.last}" }
.join("\n")
assert_operator(expected_msg, :<=, parsed_msg,
"parsed message data does not match message content:\n" +
failure_msg)
expected_msg.each do |key, value|
assert_equal(value, parsed_msg[key], "parsed message data does not match test/data/mail/#{m[:source]}: #{key}")
end
# assert: attachments in parsed message hash match metadata in raw hash
next if m[:content][:attachments].blank?

30
test/unit/email_process_test.rb
View file

@ -48,7 +48,7 @@ Some Text",
{
data: "From: me@example.com
To: customer@example.com
Subject: äöü some subject
Subject: äöü some subject 1
Some Textäöü",
channel: {
@ -58,7 +58,7 @@ Some Textäöü",
result: {
0 => {
priority: '2 normal',
title: 'äöü some subject',
title: 'äöü some subject 1',
},
1 => {
body: 'Some Textäöü',
@ -71,7 +71,7 @@ Some Textäöü",
{
data: "From: me@exampl'e.com
To: customer@exampl'e.com
Subject: äöü some subject
Subject: äöü some subject 2
Some Textäöü",
channel: {
@ -81,7 +81,7 @@ Some Textäöü",
result: {
0 => {
priority: '2 normal',
title: 'äöü some subject',
title: 'äöü some subject 2',
},
1 => {
body: 'Some Textäöü',
@ -155,17 +155,17 @@ Some Textäöü without subject#2",
{
data: "From: me@example.com
To: customer@example.com
Subject: äöü some subject
Subject: äöü some subject 3
Some Textäöü".encode('ISO-8859-1'),
success: true,
result: {
0 => {
priority: '2 normal',
title: 'äöü some subject',
title: '??? some subject 3', # it's ok, because subject need to be 7bit encoded
},
1 => {
body: 'Some Textäöü',
body: 'Some Text???', # it's ok, because no content-type is given
sender: 'Customer',
type: 'email',
internal: false,
@ -627,7 +627,7 @@ Some Text",
result: {
0 => {
priority: '2 normal',
title: 'Subject: 【专业为您注册香港及海外公司（好处多多）】　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　',
title: '【专业为您注册香港及海外公司（好处多多）】　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　',
},
1 => {
body: 'Some Text',
@ -2758,7 +2758,7 @@ Some Text',
title: '转发：整体提升企业服务水平',
},
1 => {
from: '"武兰成" <Glopelf7121@example.com>',
from: '"ÎäŔźłÉ" <Glopelf7121@example.com>',
sender: 'Customer',
type: 'email',
},
@ -2766,9 +2766,9 @@ Some Text',
verify: {
users: [
{
firstname: '武兰成',
firstname: 'ÎäŔźłÉ',
lastname: '',
fullname: '武兰成',
fullname: 'ÎäŔźłÉ',
email: 'glopelf7121@example.com',
},
],
@ -2990,7 +2990,7 @@ Some Text',
title: 'some subject 3',
},
1 => {
from: '"Vandromme, Frédéric" <fvandromme@example.com>',
from: '=?windows-1258?B?VmFuZHJvbW1lLCBGculk6XJpYw==?= <fvandromme@example.com>',
sender: 'Customer',
type: 'email',
body: 'Some Text',
@ -3008,8 +3008,7 @@ Some Text',
},
},
{
data: <<~RAW_MAIL.chomp,
From: me@example.com
data: 'From: me@example.com
To: customer@example.com
Subject: some subject
Content-Type: text/html; charset=us-ascii; format=flowed
@ -3018,8 +3017,7 @@ Some Text',
<body>
<a href="mailto:testäöü@example.com">test</a>
</body>
</html>
RAW_MAIL
</html>',
success: true,
result: {
0 => {

5
test/unit/ticket_test.rb
View file

@ -2,6 +2,11 @@
require 'test_helper'
class TicketTest < ActiveSupport::TestCase
setup do
Ticket.destroy_all
end
test 'ticket create' do
ticket = Ticket.create!(
title: "some title\n äöüß",