From 28b5f09b8e9da96963d7c4d8ef3aa93b1b36322a Mon Sep 17 00:00:00 2001 From: Martin Edenhofer Date: Fri, 4 May 2012 21:30:22 +0200 Subject: [PATCH] Improved charset handling. --- app/models/channel/email_parser.rb | 25 +++++++++++++++++-------- test/fixtures/mail2.box | 17 +++++++++++++++++ test/unit/email_parser_test.rb | 23 ++++++++++++++++++++--- 3 files changed, 54 insertions(+), 11 deletions(-) create mode 100644 test/fixtures/mail2.box diff --git a/app/models/channel/email_parser.rb b/app/models/channel/email_parser.rb index 28f95dc7c..a5e882395 100644 --- a/app/models/channel/email_parser.rb +++ b/app/models/channel/email_parser.rb @@ -2,10 +2,14 @@ require 'mail' require 'iconv' class Channel::EmailParser def conv (charset, string) - if charset == 'US-ASCII' then + if charset == 'US-ASCII' || charset == 'ASCII-8BIT' charset = 'LATIN1' end - Iconv.conv("UTF8", charset, string) + return string if charset.downcase == 'utf8' || charset.downcase == 'utf-8' +# puts '-------' + charset +# puts string +# string.encode("UTF-8") + Iconv.conv( 'UTF8', charset, string ) end def parse (msg) @@ -16,17 +20,22 @@ class Channel::EmailParser data[:from_email] = Mail::Address.new( mail[:from].value ).address data[:from_display_name] = Mail::Address.new( mail[:from].value ).display_name ['from', 'to', 'cc', 'subject'].each {|key| - data[key.to_sym] = mail[key] ? conv( mail[key].charset || 'LATIN1', mail[key].to_s) : nil + data[key.to_sym] = mail[key] ? mail[key].to_s : nil } # message id data[:message_id] = mail['message_id'] ? mail['message_id'].to_s : nil # body - # plain_part = mail.multipart? ? (mail.text_part ? mail.text_part.body.decoded : nil) : mail.body.decoded - # html_part = message.html_part ? message.html_part.body.decoded : nil - data[:plain_part] = mail.multipart? ? (mail.text_part ? mail.text_part.body.decoded : nil) : mail.body.decoded - data[:plain_part] = conv( mail.body.charset || 'LATIN1', data[:plain_part] ) +# plain_part = mail.multipart? ? (mail.text_part ? mail.text_part.body.decoded : nil) : mail.body.decoded +# html_part = message.html_part ? message.html_part.body.decoded : nil + if mail.multipart? + data[:plain_part] = mail.text_part.body.decoded + data[:plain_part] = conv( mail.text_part.charset || 'LATIN1', data[:plain_part] ) + else + data[:plain_part] = mail.body.decoded + data[:plain_part] = conv( mail.body.charset || 'LATIN1', data[:plain_part] ) + end # attachments if mail.attachments @@ -100,7 +109,7 @@ class Channel::EmailParser # create new ticket if !ticket then ticket = Ticket.create( - :group_id => channel[:group_id], + :group_id => channel[:group_id] || 1, :customer_id => user.id, :title => mail[:subject], :ticket_state_id => Ticket::State.where(:name => 'new').first.id, diff --git a/test/fixtures/mail2.box b/test/fixtures/mail2.box new file mode 100644 index 000000000..61af065b9 --- /dev/null +++ b/test/fixtures/mail2.box @@ -0,0 +1,17 @@ +From: Martin Edenhofer +Content-Type: text/plain; + charset=iso-8859-1 +Content-Transfer-Encoding: quoted-printable +Subject: =?iso-8859-1?Q?aa=E4=F6=FC=DFad_asd?= +X-Universally-Unique-Identifier: d12c15d2-e6d6-4ccd-86c7-abc2c3d0a2a2 +Date: Fri, 4 May 2012 14:01:03 +0200 +Message-Id: +To: metest@znuny.com +Mime-Version: 1.0 (Apple Message framework v1257) + +=E4=F6=FC=DF ad asd + +-Martin + +-- +Old programmers never die. They just branch to a new address. diff --git a/test/unit/email_parser_test.rb b/test/unit/email_parser_test.rb index e2a47b0c8..89032c365 100644 --- a/test/unit/email_parser_test.rb +++ b/test/unit/email_parser_test.rb @@ -14,6 +14,17 @@ class EmailParserTest < ActiveSupport::TestCase :subject => 'CI Daten für PublicView ', }, }, + { + :data => IO.read('test/fixtures/mail2.box'), + :body_md5 => '25a1ff722497271965b55e52659784a6', + :params => { + :from => 'Martin Edenhofer ', + :from_email => 'martin@example.com', + :from_display_name => 'Martin Edenhofer', + :subject => 'aaäöüßad asd', + :plain_part => "äöüß ad asd\r\n\r\n-Martin\r\n\r\n--\r\nOld programmers never die. They just branch to a new address.", + }, + }, ] files.each { |file| @@ -21,11 +32,17 @@ class EmailParserTest < ActiveSupport::TestCase parser = Channel::EmailParser.new data = parser.parse( file[:data] ) - # create md5 of body + # check body md5 = Digest::MD5.hexdigest( data[:plain_part] ) - assert_equal( file[:body_md5], md5 ) + assert_equal( file[:body_md5], md5 ) + + # check params file[:params].each { |key, value| - assert_equal( file[:params][key.to_sym], data[key.to_sym] ) + if key.to_s == 'plain_part' + assert_equal( Digest::MD5.hexdigest( file[:params][key.to_sym].to_s ), Digest::MD5.hexdigest( data[key.to_sym].to_s ) ) + else + assert_equal( file[:params][key.to_sym], data[key.to_sym] ) + end } } end