1029 lines
32 KiB
Ruby
1029 lines
32 KiB
Ruby
# Copyright (C) 2012-2021 Zammad Foundation, http://zammad-foundation.org/
|
|
|
|
# encoding: utf-8
|
|
|
|
class Channel::EmailParser
|
|
PROZESS_TIME_MAX = 180
|
|
EMAIL_REGEX = %r{.+@.+}.freeze
|
|
RECIPIENT_FIELDS = %w[to cc delivered-to x-original-to envelope-to].freeze
|
|
SENDER_FIELDS = %w[from reply-to return-path sender].freeze
|
|
EXCESSIVE_LINKS_MSG = 'This message cannot be displayed because it contains over 5,000 links. Download the raw message below and open it via an Email client if you still wish to view it.'.freeze
|
|
|
|
=begin
|
|
|
|
parser = Channel::EmailParser.new
|
|
mail = parser.parse(msg_as_string)
|
|
|
|
mail = {
|
|
from: 'Some Name <some@example.com>',
|
|
from_email: 'some@example.com',
|
|
from_local: 'some',
|
|
from_domain: 'example.com',
|
|
from_display_name: 'Some Name',
|
|
message_id: 'some_message_id@example.com',
|
|
to: 'Some System <system@example.com>',
|
|
cc: 'Somebody <somebody@example.com>',
|
|
subject: 'some message subject',
|
|
body: 'some message body',
|
|
content_type: 'text/html', # text/plain
|
|
date: Time.zone.now,
|
|
attachments: [
|
|
{
|
|
data: 'binary of attachment',
|
|
filename: 'file_name_of_attachment.txt',
|
|
preferences: {
|
|
'content-alternative' => true,
|
|
'Mime-Type' => 'text/plain',
|
|
'Charset: => 'iso-8859-1',
|
|
},
|
|
},
|
|
],
|
|
|
|
# ignore email header
|
|
x-zammad-ignore: 'false',
|
|
|
|
# customer headers
|
|
x-zammad-customer-login: '',
|
|
x-zammad-customer-email: '',
|
|
x-zammad-customer-firstname: '',
|
|
x-zammad-customer-lastname: '',
|
|
|
|
# ticket headers (for new tickets)
|
|
x-zammad-ticket-group: 'some_group',
|
|
x-zammad-ticket-state: 'some_state',
|
|
x-zammad-ticket-priority: 'some_priority',
|
|
x-zammad-ticket-owner: 'some_owner_login',
|
|
|
|
# ticket headers (for existing tickets)
|
|
x-zammad-ticket-followup-group: 'some_group',
|
|
x-zammad-ticket-followup-state: 'some_state',
|
|
x-zammad-ticket-followup-priority: 'some_priority',
|
|
x-zammad-ticket-followup-owner: 'some_owner_login',
|
|
|
|
# article headers
|
|
x-zammad-article-internal: false,
|
|
x-zammad-article-type: 'agent',
|
|
x-zammad-article-sender: 'customer',
|
|
|
|
# all other email headers
|
|
some-header: 'some_value',
|
|
}
|
|
|
|
=end
|
|
|
|
def parse(msg)
|
|
msg = msg.force_encoding('binary')
|
|
# mail 2.6 and earlier accepted non-conforming mails that lacked the correct CRLF seperators,
|
|
# mail 2.7 and above require CRLF so we force it on using binary_unsafe_to_crlf
|
|
msg = Mail::Utilities.binary_unsafe_to_crlf(msg)
|
|
mail = Mail.new(msg)
|
|
|
|
message_ensure_message_id(msg, mail)
|
|
|
|
force_parts_encoding_if_needed(mail)
|
|
|
|
headers = message_header_hash(mail)
|
|
body = message_body_hash(mail)
|
|
message_attributes = [
|
|
{ mail_instance: mail },
|
|
headers,
|
|
body,
|
|
self.class.sender_attributes(headers),
|
|
{ raw: msg },
|
|
]
|
|
message_attributes.reduce({}.with_indifferent_access, &:merge)
|
|
end
|
|
|
|
=begin
|
|
|
|
parser = Channel::EmailParser.new
|
|
ticket, article, user, mail = parser.process(channel, email_raw_string)
|
|
|
|
returns
|
|
|
|
[ticket, article, user, mail]
|
|
|
|
do not raise an exception - e. g. if used by scheduler
|
|
|
|
parser = Channel::EmailParser.new
|
|
ticket, article, user, mail = parser.process(channel, email_raw_string, false)
|
|
|
|
returns
|
|
|
|
[ticket, article, user, mail] || false
|
|
|
|
=end
|
|
|
|
def process(channel, msg, exception = true)
|
|
|
|
Timeout.timeout(PROZESS_TIME_MAX) do
|
|
_process(channel, msg)
|
|
end
|
|
rescue => e
|
|
# store unprocessable email for bug reporting
|
|
filename = archive_mail('unprocessable_mail', msg)
|
|
|
|
message = "Can't process email, you will find it for bug reporting under #{filename}, please create an issue at https://github.com/zammad/zammad/issues"
|
|
|
|
p "ERROR: #{message}" # rubocop:disable Rails/Output
|
|
p "ERROR: #{e.inspect}" # rubocop:disable Rails/Output
|
|
Rails.logger.error message
|
|
Rails.logger.error e
|
|
|
|
return false if exception == false
|
|
|
|
raise %(#{e.inspect}\n#{e.backtrace.join("\n")})
|
|
end
|
|
|
|
def _process(channel, msg)
|
|
|
|
# parse email
|
|
mail = parse(msg)
|
|
|
|
Rails.logger.info "Process email with msgid '#{mail[:message_id]}'"
|
|
|
|
# run postmaster pre filter
|
|
UserInfo.current_user_id = 1
|
|
|
|
# set interface handle
|
|
original_interface_handle = ApplicationHandleInfo.current
|
|
transaction_params = { interface_handle: "#{original_interface_handle}.postmaster", disable: [] }
|
|
|
|
filters = {}
|
|
Setting.where(area: 'Postmaster::PreFilter').order(:name).each do |setting|
|
|
filters[setting.name] = Setting.get(setting.name).constantize
|
|
end
|
|
filters.each do |key, backend|
|
|
Rails.logger.debug { "run postmaster pre filter #{key}: #{backend}" }
|
|
begin
|
|
backend.run(channel, mail, transaction_params)
|
|
rescue => e
|
|
Rails.logger.error "can't run postmaster pre filter #{key}: #{backend}"
|
|
Rails.logger.error e.inspect
|
|
raise e
|
|
end
|
|
end
|
|
|
|
# check ignore header
|
|
if mail[:'x-zammad-ignore'] == 'true' || mail[:'x-zammad-ignore'] == true
|
|
Rails.logger.info "ignored email with msgid '#{mail[:message_id]}' from '#{mail[:from]}' because of x-zammad-ignore header"
|
|
return
|
|
end
|
|
|
|
ticket = nil
|
|
article = nil
|
|
session_user = nil
|
|
|
|
# use transaction
|
|
Transaction.execute(transaction_params) do
|
|
|
|
# get sender user
|
|
session_user_id = mail[:'x-zammad-session-user-id']
|
|
if !session_user_id
|
|
raise 'No x-zammad-session-user-id, no sender set!'
|
|
end
|
|
|
|
session_user = User.lookup(id: session_user_id)
|
|
if !session_user
|
|
raise "No user found for x-zammad-session-user-id: #{session_user_id}!"
|
|
end
|
|
|
|
# set current user
|
|
UserInfo.current_user_id = session_user.id
|
|
|
|
# get ticket# based on email headers
|
|
if mail[:'x-zammad-ticket-id']
|
|
ticket = Ticket.find_by(id: mail[:'x-zammad-ticket-id'])
|
|
end
|
|
if mail[:'x-zammad-ticket-number']
|
|
ticket = Ticket.find_by(number: mail[:'x-zammad-ticket-number'])
|
|
end
|
|
|
|
# set ticket state to open if not new
|
|
if ticket
|
|
set_attributes_by_x_headers(ticket, 'ticket', mail, 'followup')
|
|
|
|
# save changes set by x-zammad-ticket-followup-* headers
|
|
ticket.save! if ticket.has_changes_to_save?
|
|
|
|
# set ticket to open again or keep create state
|
|
if !mail[:'x-zammad-ticket-followup-state'] && !mail[:'x-zammad-ticket-followup-state_id']
|
|
new_state = Ticket::State.find_by(default_create: true)
|
|
if ticket.state_id != new_state.id && !mail[:'x-zammad-out-of-office']
|
|
ticket.state = Ticket::State.find_by(default_follow_up: true)
|
|
ticket.save!
|
|
end
|
|
end
|
|
end
|
|
|
|
# create new ticket
|
|
if !ticket
|
|
|
|
preferences = {}
|
|
if channel[:id]
|
|
preferences = {
|
|
channel_id: channel[:id]
|
|
}
|
|
end
|
|
|
|
# get default group where ticket is created
|
|
group = nil
|
|
if channel[:group_id]
|
|
group = Group.lookup(id: channel[:group_id])
|
|
else
|
|
mail_to_group = self.class.mail_to_group(mail[:to])
|
|
if mail_to_group.present?
|
|
group = mail_to_group
|
|
end
|
|
end
|
|
if group.blank? || group.active == false
|
|
group = Group.where(active: true).order(id: :asc).first
|
|
end
|
|
if group.blank?
|
|
group = Group.first
|
|
end
|
|
title = mail[:subject]
|
|
if title.blank?
|
|
title = '-'
|
|
end
|
|
ticket = Ticket.new(
|
|
group_id: group.id,
|
|
title: title,
|
|
preferences: preferences,
|
|
)
|
|
set_attributes_by_x_headers(ticket, 'ticket', mail)
|
|
|
|
# create ticket
|
|
ticket.save!
|
|
|
|
end
|
|
|
|
# apply tags to ticket
|
|
if mail[:'x-zammad-ticket-tags'].present?
|
|
mail[:'x-zammad-ticket-tags'].each do |tag|
|
|
ticket.tag_add(tag)
|
|
end
|
|
end
|
|
|
|
# set attributes
|
|
ticket.with_lock do
|
|
article = Ticket::Article.new(
|
|
ticket_id: ticket.id,
|
|
type_id: Ticket::Article::Type.find_by(name: 'email').id,
|
|
sender_id: Ticket::Article::Sender.find_by(name: 'Customer').id,
|
|
content_type: mail[:content_type],
|
|
body: mail[:body],
|
|
from: mail[:from],
|
|
reply_to: mail[:'reply-to'],
|
|
to: mail[:to],
|
|
cc: mail[:cc],
|
|
subject: mail[:subject],
|
|
message_id: mail[:message_id],
|
|
internal: false,
|
|
)
|
|
|
|
# x-headers lookup
|
|
set_attributes_by_x_headers(article, 'article', mail)
|
|
|
|
# create article
|
|
article.save!
|
|
|
|
# store mail plain
|
|
article.save_as_raw(msg)
|
|
|
|
# store attachments
|
|
mail[:attachments]&.each do |attachment|
|
|
filename = attachment[:filename].force_encoding('utf-8')
|
|
if !filename.force_encoding('UTF-8').valid_encoding?
|
|
filename = filename.utf8_encode(fallback: :read_as_sanitized_binary)
|
|
end
|
|
Store.add(
|
|
object: 'Ticket::Article',
|
|
o_id: article.id,
|
|
data: attachment[:data],
|
|
filename: filename,
|
|
preferences: attachment[:preferences]
|
|
)
|
|
end
|
|
end
|
|
end
|
|
|
|
ticket.reload
|
|
article.reload
|
|
session_user.reload
|
|
|
|
# run postmaster post filter
|
|
filters = {}
|
|
Setting.where(area: 'Postmaster::PostFilter').order(:name).each do |setting|
|
|
filters[setting.name] = Setting.get(setting.name).constantize
|
|
end
|
|
filters.each_value do |backend|
|
|
Rails.logger.debug { "run postmaster post filter #{backend}" }
|
|
begin
|
|
backend.run(channel, mail, ticket, article, session_user)
|
|
rescue => e
|
|
Rails.logger.error "can't run postmaster post filter #{backend}"
|
|
Rails.logger.error e.inspect
|
|
end
|
|
end
|
|
|
|
# return new objects
|
|
[ticket, article, session_user, mail]
|
|
end
|
|
|
|
def self.mail_to_group(to)
|
|
begin
|
|
to = Mail::AddressList.new(to)&.addresses&.first&.address
|
|
rescue
|
|
Rails.logger.error 'Can not parse :to field for group destination!'
|
|
end
|
|
return if to.blank?
|
|
|
|
email = EmailAddress.find_by(email: to.downcase)
|
|
return if email&.channel.blank?
|
|
|
|
email.channel&.group
|
|
end
|
|
|
|
def self.check_attributes_by_x_headers(header_name, value)
|
|
class_name = nil
|
|
attribute = nil
|
|
# skip check attributes if it is tags
|
|
return true if header_name == 'x-zammad-ticket-tags'
|
|
|
|
if header_name =~ %r{^x-zammad-(.+?)-(followup-|)(.*)$}i
|
|
class_name = $1
|
|
attribute = $3
|
|
end
|
|
return true if !class_name
|
|
|
|
if class_name.casecmp('article').zero?
|
|
class_name = 'Ticket::Article'
|
|
end
|
|
return true if !attribute
|
|
|
|
key_short = attribute[ attribute.length - 3, attribute.length ]
|
|
return true if key_short != '_id'
|
|
|
|
class_object = class_name.to_classname.constantize
|
|
return if !class_object
|
|
|
|
class_instance = class_object.new
|
|
|
|
return false if !class_instance.association_id_validation(attribute, value)
|
|
|
|
true
|
|
end
|
|
|
|
def self.sender_attributes(from)
|
|
if from.is_a?(HashWithIndifferentAccess)
|
|
from = SENDER_FIELDS.filter_map { |f| from[f] }
|
|
.map(&:to_utf8).reject(&:blank?)
|
|
.partition { |address| address.match?(EMAIL_REGEX) }
|
|
.flatten.first
|
|
end
|
|
|
|
data = {}.with_indifferent_access
|
|
return data if from.blank?
|
|
|
|
from = from.gsub('<>', '').strip
|
|
mail_address = begin
|
|
Mail::AddressList.new(from).addresses
|
|
.select { |a| a.address.present? }
|
|
.partition { |a| a.address.match?(EMAIL_REGEX) }
|
|
.flatten.first
|
|
rescue Mail::Field::ParseError => e
|
|
$stdout.puts e
|
|
end
|
|
|
|
if mail_address&.address.present?
|
|
data[:from_email] = mail_address.address
|
|
data[:from_local] = mail_address.local
|
|
data[:from_domain] = mail_address.domain
|
|
data[:from_display_name] = mail_address.display_name || mail_address.comments&.first
|
|
elsif from =~ %r{^(.+?)<((.+?)@(.+?))>}
|
|
data[:from_email] = $2
|
|
data[:from_local] = $3
|
|
data[:from_domain] = $4
|
|
data[:from_display_name] = $1
|
|
else
|
|
data[:from_email] = from
|
|
data[:from_local] = from
|
|
data[:from_domain] = from
|
|
data[:from_display_name] = from
|
|
end
|
|
|
|
# do extra decoding because we needed to use field.value
|
|
data[:from_display_name] =
|
|
Mail::Field.new('X-From', data[:from_display_name].to_utf8)
|
|
.to_s
|
|
.delete('"')
|
|
.strip
|
|
.gsub(%r{(^'|'$)}, '')
|
|
|
|
data
|
|
end
|
|
|
|
def set_attributes_by_x_headers(item_object, header_name, mail, suffix = false)
|
|
|
|
# loop all x-zammad-header-* headers
|
|
item_object.attributes.each_key do |key|
|
|
|
|
# ignore read only attributes
|
|
next if key == 'updated_by_id'
|
|
next if key == 'created_by_id'
|
|
|
|
# check if id exists
|
|
key_short = key[ key.length - 3, key.length ]
|
|
if key_short == '_id'
|
|
key_short = key[ 0, key.length - 3 ]
|
|
header = "x-zammad-#{header_name}-#{key_short}"
|
|
if suffix
|
|
header = "x-zammad-#{header_name}-#{suffix}-#{key_short}"
|
|
end
|
|
|
|
# only set value on _id if value/reference lookup exists
|
|
if mail[header.to_sym]
|
|
|
|
Rails.logger.info "set_attributes_by_x_headers header #{header} found #{mail[header.to_sym]}"
|
|
item_object.class.reflect_on_all_associations.map do |assoc|
|
|
|
|
next if assoc.name.to_s != key_short
|
|
|
|
Rails.logger.info "set_attributes_by_x_headers found #{assoc.class_name} lookup for '#{mail[header.to_sym]}'"
|
|
item = assoc.class_name.constantize
|
|
assoc_object = nil
|
|
if item.new.respond_to?(:name)
|
|
assoc_object = item.lookup(name: mail[header.to_sym])
|
|
end
|
|
if !assoc_object && item.new.respond_to?(:login)
|
|
assoc_object = item.lookup(login: mail[header.to_sym])
|
|
end
|
|
if !assoc_object && item.new.respond_to?(:email)
|
|
assoc_object = item.lookup(email: mail[header.to_sym])
|
|
end
|
|
|
|
if assoc_object.blank?
|
|
|
|
# no assoc exists, remove header
|
|
mail.delete(header.to_sym)
|
|
next
|
|
end
|
|
|
|
Rails.logger.info "set_attributes_by_x_headers assign #{item_object.class} #{key}=#{assoc_object.id}"
|
|
|
|
item_object[key] = assoc_object.id
|
|
|
|
end
|
|
end
|
|
end
|
|
|
|
# check if attribute exists
|
|
header = "x-zammad-#{header_name}-#{key}"
|
|
if suffix
|
|
header = "x-zammad-#{header_name}-#{suffix}-#{key}"
|
|
end
|
|
if mail[header.to_sym]
|
|
Rails.logger.info "set_attributes_by_x_headers header #{header} found. Assign #{key}=#{mail[header.to_sym]}"
|
|
item_object[key] = mail[header.to_sym]
|
|
end
|
|
end
|
|
end
|
|
|
|
=begin
|
|
|
|
process unprocessable_mails (tmp/unprocessable_mail/*.eml) again
|
|
|
|
Channel::EmailParser.process_unprocessable_mails
|
|
|
|
=end
|
|
|
|
def self.process_unprocessable_mails(params = {})
|
|
path = Rails.root.join('tmp/unprocessable_mail')
|
|
files = []
|
|
Dir.glob("#{path}/*.eml") do |entry|
|
|
ticket, _article, _user, _mail = Channel::EmailParser.new.process(params, IO.binread(entry))
|
|
next if ticket.blank?
|
|
|
|
files.push entry
|
|
File.delete(entry)
|
|
end
|
|
files
|
|
end
|
|
|
|
=begin
|
|
|
|
process oversized emails by:
|
|
1. Archiving the oversized mail as tmp/oversized_mail/md5.eml
|
|
2. Reply with a postmaster message to inform the sender
|
|
|
|
=end
|
|
|
|
def process_oversized_mail(channel, msg)
|
|
archive_mail('oversized_mail', msg)
|
|
postmaster_response(channel, msg)
|
|
end
|
|
|
|
private
|
|
|
|
# https://github.com/zammad/zammad/issues/2922
|
|
def force_parts_encoding_if_needed(mail)
|
|
# enforce encoding on both multipart parts and main body
|
|
([mail] + mail.parts).each { |elem| force_single_part_encoding_if_needed(elem) }
|
|
end
|
|
|
|
# https://github.com/zammad/zammad/issues/2922
|
|
def force_single_part_encoding_if_needed(part)
|
|
return if part.charset&.downcase != 'iso-2022-jp'
|
|
|
|
part.body = force_japanese_encoding part.body.encoded.unpack1('M')
|
|
end
|
|
|
|
ISO2022JP_REGEXP = %r{=\?ISO-2022-JP\?B\?(.+?)\?=}.freeze
|
|
|
|
# https://github.com/zammad/zammad/issues/3115
|
|
def header_field_unpack_japanese(field)
|
|
field.value.gsub ISO2022JP_REGEXP do
|
|
force_japanese_encoding Base64.decode64($1)
|
|
end
|
|
end
|
|
|
|
# generate Message ID on the fly if it was missing
|
|
# yes, Mail gem generates one in some cases
|
|
# but it is 100% random so duplicate messages would not be detected
|
|
def message_ensure_message_id(raw, parsed)
|
|
field = parsed.header.fields.find { |elem| elem.name == 'Message-ID' }
|
|
|
|
return true if field&.unparsed_value.present?
|
|
|
|
parsed.message_id = generate_message_id(raw, parsed.from)
|
|
end
|
|
|
|
def message_header_hash(mail)
|
|
imported_fields = mail.header.fields.map do |f|
|
|
begin
|
|
value = if f.value.match?(ISO2022JP_REGEXP)
|
|
value = header_field_unpack_japanese(f)
|
|
else
|
|
f.decoded.to_utf8
|
|
end
|
|
# fields that cannot be cleanly parsed fallback to the empty string
|
|
rescue Mail::Field::IncompleteParseError
|
|
value = ''
|
|
rescue Encoding::CompatibilityError => e
|
|
try_iso88591 = f.value.force_encoding('iso-8859-1').encode('utf-8')
|
|
|
|
raise e if !try_iso88591.is_utf8?
|
|
|
|
f.value = try_iso88591
|
|
value = f.decoded.to_utf8
|
|
rescue
|
|
value = f.decoded.to_utf8(fallback: :read_as_sanitized_binary)
|
|
end
|
|
[f.name.downcase, value]
|
|
end.to_h
|
|
|
|
# imported_fields = mail.header.fields.map { |f| [f.name.downcase, f.to_utf8] }.to_h
|
|
raw_fields = mail.header.fields.index_by { |f| "raw-#{f.name.downcase}" }
|
|
custom_fields = {}.tap do |h|
|
|
h.replace(imported_fields.slice(*RECIPIENT_FIELDS)
|
|
.transform_values { |v| v.match?(EMAIL_REGEX) ? v : '' })
|
|
|
|
h['x-any-recipient'] = h.values.select(&:present?).join(', ')
|
|
h['message_id'] = imported_fields['message-id']
|
|
h['subject'] = imported_fields['subject']
|
|
begin
|
|
h['date'] = Time.zone.parse(mail.date.to_s) || imported_fields['date']
|
|
rescue
|
|
h['date'] = nil
|
|
end
|
|
end
|
|
|
|
[imported_fields, raw_fields, custom_fields].reduce({}.with_indifferent_access, &:merge)
|
|
end
|
|
|
|
def message_body_hash(mail)
|
|
if mail.html_part&.body.present?
|
|
content_type = mail.html_part.mime_type || 'text/plain'
|
|
body = body_text(mail.html_part, strict_html: true)
|
|
elsif mail.text_part.present? && mail.all_parts.any? { |elem| elem.inline? && elem.content_type&.start_with?('image') }
|
|
content_type = 'text/html'
|
|
|
|
body = mail
|
|
.all_parts
|
|
.reduce('') do |memo, part|
|
|
if part.mime_type == 'text/plain' && !part.attachment?
|
|
memo += body_text(part, strict_html: false).text2html
|
|
elsif part.inline? && part.content_type&.start_with?('image')
|
|
memo += "<img src=\'cid:#{part.cid}\'>"
|
|
end
|
|
|
|
memo
|
|
end
|
|
elsif mail.text_part.present?
|
|
content_type = 'text/plain'
|
|
|
|
body = mail
|
|
.all_parts
|
|
.reduce('') do |memo, part|
|
|
if part.mime_type == 'text/plain' && !part.attachment?
|
|
memo += body_text(part, strict_html: false)
|
|
end
|
|
|
|
memo
|
|
end
|
|
elsif mail&.body.present? && (mail.mime_type.nil? || mail.mime_type.match?(%r{^text/(plain|html)$}))
|
|
content_type = mail.mime_type || 'text/plain'
|
|
body = body_text(mail, strict_html: content_type.eql?('text/html'))
|
|
end
|
|
|
|
content_type = 'text/plain' if body.blank?
|
|
|
|
{
|
|
attachments: collect_attachments(mail),
|
|
content_type: content_type || 'text/plain',
|
|
body: body.presence || 'no visible content'
|
|
}.with_indifferent_access
|
|
end
|
|
|
|
def body_text(message, **options)
|
|
body_text = begin
|
|
message.body.to_s
|
|
rescue Mail::UnknownEncodingType # see test/data/mail/mail043.box / issue #348
|
|
message.body.raw_source
|
|
end
|
|
|
|
body_text = body_text.utf8_encode(from: message.charset, fallback: :read_as_sanitized_binary)
|
|
body_text = Mail::Utilities.to_lf(body_text)
|
|
|
|
# plaintext body requires no processing
|
|
return body_text if !options[:strict_html]
|
|
|
|
# Issue #2390 - emails with >5k HTML links should be rejected
|
|
return EXCESSIVE_LINKS_MSG if body_text.scan(%r{<a[[:space:]]}i).count >= 5_000
|
|
|
|
body_text.html2html_strict
|
|
end
|
|
|
|
def collect_attachments(mail)
|
|
attachments = []
|
|
|
|
attachments.push(*get_nonplaintext_body_as_attachment(mail))
|
|
|
|
mail.parts.each do |part|
|
|
attachments.push(*gracefully_get_attachments(part, attachments, mail))
|
|
end
|
|
|
|
attachments
|
|
end
|
|
|
|
def get_nonplaintext_body_as_attachment(mail)
|
|
if !(mail.html_part&.body.present? || (!mail.multipart? && mail.mime_type.present? && mail.mime_type != 'text/plain'))
|
|
return
|
|
end
|
|
|
|
message = mail.html_part || mail
|
|
|
|
if !mail.mime_type.starts_with?('text/') && mail.html_part.blank?
|
|
return gracefully_get_attachments(message, [], mail)
|
|
end
|
|
|
|
filename = message.filename.presence || (message.mime_type.eql?('text/html') ? 'message.html' : '-no name-')
|
|
|
|
headers_store = {
|
|
'content-alternative' => true,
|
|
'original-format' => message.mime_type.eql?('text/html'),
|
|
'Mime-Type' => message.mime_type,
|
|
'Charset' => message.charset,
|
|
}.reject { |_, v| v.blank? }
|
|
|
|
[{
|
|
data: body_text(message),
|
|
filename: filename,
|
|
preferences: headers_store
|
|
}]
|
|
end
|
|
|
|
def gracefully_get_attachments(part, attachments, mail)
|
|
get_attachments(part, attachments, mail).flatten.compact
|
|
rescue => e # Protect process to work with spam emails (see test/fixtures/mail15.box)
|
|
raise e if (fail_count ||= 0).positive?
|
|
|
|
(fail_count += 1) && retry
|
|
end
|
|
|
|
def get_attachments(file, attachments, mail)
|
|
return file.parts.map { |p| get_attachments(p, attachments, mail) } if file.parts.any?
|
|
return [] if [mail.text_part&.body&.encoded, mail.html_part&.body&.encoded].include?(file.body.encoded)
|
|
return [] if file.content_type&.start_with?('text/plain') && !file.attachment?
|
|
|
|
# get file preferences
|
|
headers_store = {}
|
|
file.header.fields.each do |field|
|
|
|
|
# full line, encode, ready for storage
|
|
value = field.to_utf8
|
|
if value.blank?
|
|
value = field.raw_value
|
|
end
|
|
headers_store[field.name.to_s] = value
|
|
rescue
|
|
headers_store[field.name.to_s] = field.raw_value
|
|
end
|
|
|
|
# cleanup content id, <> will be added automatically later
|
|
if headers_store['Content-ID'].blank? && headers_store['Content-Id'].present?
|
|
headers_store['Content-ID'] = headers_store['Content-Id']
|
|
end
|
|
if headers_store['Content-ID']
|
|
headers_store['Content-ID'].delete_prefix!('<')
|
|
headers_store['Content-ID'].delete_suffix!('>')
|
|
end
|
|
|
|
# get filename from content-disposition
|
|
|
|
# workaround for: NoMethodError: undefined method `filename' for #<Mail::UnstructuredField:0x007ff109e80678>
|
|
begin
|
|
filename = file.header[:content_disposition].try(:filename)
|
|
rescue
|
|
begin
|
|
case file.header[:content_disposition].to_s
|
|
when %r{(filename|name)(\*{0,1})="(.+?)"}i, %r{(filename|name)(\*{0,1})='(.+?)'}i, %r{(filename|name)(\*{0,1})=(.+?);}i
|
|
filename = $3
|
|
end
|
|
rescue
|
|
Rails.logger.debug { 'Unable to get filename' }
|
|
end
|
|
end
|
|
|
|
begin
|
|
case file.header[:content_disposition].to_s
|
|
when %r{(filename|name)(\*{0,1})="(.+?)"}i, %r{(filename|name)(\*{0,1})='(.+?)'}i, %r{(filename|name)(\*{0,1})=(.+?);}i
|
|
filename = $3
|
|
end
|
|
rescue
|
|
Rails.logger.debug { 'Unable to get filename' }
|
|
end
|
|
|
|
# as fallback, use raw values
|
|
if filename.blank?
|
|
case headers_store['Content-Disposition'].to_s
|
|
when %r{(filename|name)(\*{0,1})="(.+?)"}i, %r{(filename|name)(\*{0,1})='(.+?)'}i, %r{(filename|name)(\*{0,1})=(.+?);}i
|
|
filename = $3
|
|
end
|
|
end
|
|
|
|
# for some broken sm mail clients (X-MimeOLE: Produced By Microsoft Exchange V6.5)
|
|
filename ||= file.header[:content_location].to_s.dup.force_encoding('utf-8')
|
|
|
|
file_body = String.new(file.body.to_s)
|
|
|
|
# generate file name based on content type
|
|
if filename.blank? && headers_store['Content-Type'].present? && headers_store['Content-Type'].match?(%r{^message/rfc822}i)
|
|
begin
|
|
parser = Channel::EmailParser.new
|
|
mail_local = parser.parse(file_body)
|
|
filename = if mail_local[:subject].present?
|
|
"#{mail_local[:subject]}.eml"
|
|
elsif headers_store['Content-Description'].present?
|
|
"#{headers_store['Content-Description']}.eml".to_s.force_encoding('utf-8')
|
|
else
|
|
'Mail.eml'
|
|
end
|
|
rescue
|
|
filename = 'Mail.eml'
|
|
end
|
|
end
|
|
|
|
# e. g. Content-Type: video/quicktime; name="Video.MOV";
|
|
if filename.blank?
|
|
['(filename|name)(\*{0,1})="(.+?)"(;|$)', '(filename|name)(\*{0,1})=\'(.+?)\'(;|$)', '(filename|name)(\*{0,1})=(.+?)(;|$)'].each do |regexp|
|
|
if headers_store['Content-Type'] =~ %r{#{regexp}}i
|
|
filename = $3
|
|
break
|
|
end
|
|
end
|
|
end
|
|
|
|
# workaround for mail gem - decode filenames
|
|
# https://github.com/zammad/zammad/issues/928
|
|
if filename.present?
|
|
filename = Mail::Encodings.value_decode(filename)
|
|
end
|
|
|
|
if !filename.force_encoding('UTF-8').valid_encoding?
|
|
filename = filename.utf8_encode(fallback: :read_as_sanitized_binary)
|
|
end
|
|
|
|
# generate file name based on content-id with file extention
|
|
if filename.blank? && headers_store['Content-ID'].present? && headers_store['Content-ID'] =~ %r{(.+?\..{2,6})@.+?}i
|
|
filename = $1
|
|
end
|
|
|
|
# e. g. Content-Type: video/quicktime
|
|
if filename.blank? && (content_type = headers_store['Content-Type'])
|
|
map = {
|
|
'message/delivery-status': %w[txt delivery-status],
|
|
'text/plain': %w[txt document],
|
|
'text/html': %w[html document],
|
|
'video/quicktime': %w[mov video],
|
|
'image/jpeg': %w[jpg image],
|
|
'image/jpg': %w[jpg image],
|
|
'image/png': %w[png image],
|
|
'image/gif': %w[gif image],
|
|
}
|
|
map.each do |type, ext|
|
|
next if !content_type.match?(%r{^#{Regexp.quote(type)}}i)
|
|
|
|
filename = if headers_store['Content-Description'].present?
|
|
"#{headers_store['Content-Description']}.#{ext[0]}".to_s.force_encoding('utf-8')
|
|
else
|
|
"#{ext[1]}.#{ext[0]}"
|
|
end
|
|
break
|
|
end
|
|
end
|
|
|
|
# generate file name based on content-id without file extention
|
|
if filename.blank? && headers_store['Content-ID'].present? && headers_store['Content-ID'] =~ %r{(.+?)@.+?}i
|
|
filename = $1
|
|
end
|
|
|
|
# set fallback filename
|
|
if filename.blank?
|
|
filename = 'file'
|
|
end
|
|
|
|
# create uniq filename
|
|
local_filename = ''
|
|
local_extention = ''
|
|
if filename =~ %r{^(.*?)\.(.+?)$}
|
|
local_filename = $1
|
|
local_extention = $2
|
|
end
|
|
1.upto(1000) do |i|
|
|
filename_exists = false
|
|
attachments.each do |attachment|
|
|
if attachment[:filename] == filename
|
|
filename_exists = true
|
|
end
|
|
end
|
|
break if filename_exists == false
|
|
|
|
filename = if local_extention.present?
|
|
"#{local_filename}#{i}.#{local_extention}"
|
|
else
|
|
"#{local_filename}#{i}"
|
|
end
|
|
end
|
|
|
|
# get mime type
|
|
if file.header[:content_type]&.string
|
|
headers_store['Mime-Type'] = file.header[:content_type].string
|
|
end
|
|
|
|
# get charset
|
|
if file.header&.charset
|
|
headers_store['Charset'] = file.header.charset
|
|
end
|
|
|
|
# remove not needed header
|
|
headers_store.delete('Content-Transfer-Encoding')
|
|
headers_store.delete('Content-Disposition')
|
|
|
|
attach = {
|
|
data: file_body,
|
|
filename: filename,
|
|
preferences: headers_store,
|
|
}
|
|
|
|
[attach]
|
|
end
|
|
|
|
# Archive the given message as tmp/folder/md5.eml
|
|
def archive_mail(folder, msg)
|
|
path = Rails.root.join('tmp', folder)
|
|
FileUtils.mkpath path
|
|
|
|
# MD5 hash the msg and save it as "md5.eml"
|
|
md5 = Digest::MD5.hexdigest(msg)
|
|
file_path = Rails.root.join('tmp', folder, "#{md5}.eml")
|
|
|
|
File.open(file_path, 'wb') do |file|
|
|
file.write msg
|
|
end
|
|
|
|
file_path
|
|
end
|
|
|
|
# Auto reply as the postmaster to oversized emails with:
|
|
# [undeliverable] Message too large
|
|
def postmaster_response(channel, msg)
|
|
begin
|
|
reply_mail = compose_postmaster_reply(msg)
|
|
rescue NotificationFactory::FileNotFoundError => e
|
|
Rails.logger.error "No valid postmaster email_oversized template found. Skipping postmaster reply. #{e.inspect}"
|
|
return
|
|
end
|
|
|
|
Rails.logger.info "Send mail too large postmaster message to: #{reply_mail[:to]}"
|
|
reply_mail[:from] = EmailAddress.find_by(channel: channel).email
|
|
channel.deliver(reply_mail)
|
|
rescue => e
|
|
Rails.logger.error "Error during sending of postmaster oversized email auto-reply: #{e.inspect}\n#{e.backtrace}"
|
|
end
|
|
|
|
# Compose a "Message too large" reply to the given message
|
|
def compose_postmaster_reply(raw_incoming_mail, locale = nil)
|
|
parsed_incoming_mail = Channel::EmailParser.new.parse(raw_incoming_mail)
|
|
|
|
# construct a dummy mail object
|
|
mail = OpenStruct.new
|
|
mail.from_display_name = parsed_incoming_mail[:from_display_name]
|
|
mail.subject = parsed_incoming_mail[:subject]
|
|
mail.msg_size = format('%<MB>.2f', MB: raw_incoming_mail.size.to_f / 1024 / 1024)
|
|
|
|
reply = NotificationFactory::Mailer.template(
|
|
template: 'email_oversized',
|
|
locale: locale,
|
|
format: 'txt',
|
|
objects: {
|
|
mail: mail,
|
|
},
|
|
raw: true, # will not add application template
|
|
standalone: true, # default: false - will send header & footer
|
|
)
|
|
|
|
reply.merge(
|
|
to: parsed_incoming_mail[:from_email],
|
|
body: reply[:body].gsub(%r{\n}, "\r\n"),
|
|
content_type: 'text/plain',
|
|
References: parsed_incoming_mail[:message_id],
|
|
'In-Reply-To': parsed_incoming_mail[:message_id],
|
|
)
|
|
end
|
|
|
|
def guess_email_fqdn(from)
|
|
Mail::Address.new(from).domain.strip
|
|
rescue
|
|
nil
|
|
end
|
|
|
|
def generate_message_id(raw_message, from)
|
|
fqdn = guess_email_fqdn(from) || 'zammad_generated'
|
|
|
|
"<gen-#{Digest::MD5.hexdigest(raw_message)}@#{fqdn}>"
|
|
end
|
|
|
|
# https://github.com/zammad/zammad/issues/3096
|
|
# specific email needs to be forced to ISO-2022-JP
|
|
# but that breaks other emails that can be forced to SJIS only
|
|
# thus force to ISO-2022-JP but fallback to SJIS
|
|
#
|
|
# https://github.com/zammad/zammad/issues/3368
|
|
# some characters are not included in the official ISO-2022-JP
|
|
# ISO-2022-JP-KDDI superset provides support for more characters
|
|
def force_japanese_encoding(input)
|
|
%w[ISO-2022-JP ISO-2022-JP-KDDI SJIS]
|
|
.lazy
|
|
.map { |encoding| try_encoding(input, encoding) }
|
|
.detect(&:present?)
|
|
end
|
|
|
|
def try_encoding(input, encoding)
|
|
input.force_encoding(encoding).encode('UTF-8')
|
|
rescue
|
|
nil
|
|
end
|
|
end
|
|
|
|
module Mail
|
|
|
|
# workaround to get content of no parseable headers - in most cases with non 7 bit ascii signs
|
|
class Field
|
|
def raw_value
|
|
begin
|
|
value = @raw_value.try(:utf8_encode)
|
|
rescue
|
|
value = @raw_value.utf8_encode(fallback: :read_as_sanitized_binary)
|
|
end
|
|
return value if value.blank?
|
|
|
|
value.sub(%r{^.+?:(\s|)}, '')
|
|
end
|
|
end
|
|
|
|
# issue#348 - IMAP mail fetching stops because of broken spam email (e. g. broken Content-Transfer-Encoding value see test/fixtures/mail43.box)
|
|
# https://github.com/zammad/zammad/issues/348
|
|
class Body
|
|
def decoded
|
|
if Encodings.defined?(encoding)
|
|
Encodings.get_encoding(encoding).decode(raw_source)
|
|
else
|
|
Rails.logger.info "UnknownEncodingType: Don't know how to decode #{encoding}!"
|
|
raw_source
|
|
end
|
|
end
|
|
end
|
|
|
|
end
|