From b10ee11c86821e91fa822b505bbc25117c5918ee Mon Sep 17 00:00:00 2001 From: Martin Edenhofer Date: Wed, 19 Jul 2017 07:29:52 +0200 Subject: [PATCH] Strip null byte - postgresql will complain about it. --- app/models/application_model.rb | 2 +- .../checks_attribute_length.rb | 39 -------------- .../checks_attribute_values_and_length.rb | 54 +++++++++++++++++++ config/initializers/db_preferences_mysql.rb | 1 + .../initializers/db_preferences_postgresql.rb | 1 + lib/stats/ticket_reopen.rb | 1 + test/unit/ticket_null_byte_test.rb | 34 ++++++++++++ 7 files changed, 92 insertions(+), 40 deletions(-) delete mode 100644 app/models/application_model/checks_attribute_length.rb create mode 100644 app/models/application_model/checks_attribute_values_and_length.rb create mode 100644 test/unit/ticket_null_byte_test.rb diff --git a/app/models/application_model.rb b/app/models/application_model.rb index 551fe482b..c3d267e5f 100644 --- a/app/models/application_model.rb +++ b/app/models/application_model.rb @@ -5,7 +5,7 @@ class ApplicationModel < ActiveRecord::Base include ApplicationModel::HasCache include ApplicationModel::CanLookup include ApplicationModel::CanLookupSearchIndexAttributes - include ApplicationModel::ChecksAttributeLength + include ApplicationModel::ChecksAttributeValuesAndLength include ApplicationModel::CanCleanupParam include ApplicationModel::HasRecentViews include ApplicationModel::ChecksUserColumnsFillup diff --git a/app/models/application_model/checks_attribute_length.rb b/app/models/application_model/checks_attribute_length.rb deleted file mode 100644 index 5254b035c..000000000 --- a/app/models/application_model/checks_attribute_length.rb +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (C) 2012-2016 Zammad Foundation, http://zammad-foundation.org/ -module ApplicationModel::ChecksAttributeLength - extend ActiveSupport::Concern - - included do - before_create :check_attribute_length - before_update :check_attribute_length - end - -=begin - -check string/varchar size and cut them if needed - -=end - - def check_attribute_length - attributes.each { |attribute| - next if !self[ attribute[0] ] - next if !self[ attribute[0] ].instance_of?(String) - next if self[ attribute[0] ].empty? - column = self.class.columns_hash[ attribute[0] ] - next if !column - limit = column.limit - if column && limit - current_length = attribute[1].to_s.length - if limit < current_length - logger.warn "WARNING: cut string because of database length #{self.class}.#{attribute[0]}(#{limit} but is #{current_length}:#{attribute[1]})" - self[ attribute[0] ] = attribute[1][ 0, limit ] - end - end - - # strip 4 bytes utf8 chars if needed - if column && self[ attribute[0] ] - self[attribute[0]] = self[ attribute[0] ].utf8_to_3bytesutf8 - end - } - true - end -end diff --git a/app/models/application_model/checks_attribute_values_and_length.rb b/app/models/application_model/checks_attribute_values_and_length.rb new file mode 100644 index 000000000..27a680866 --- /dev/null +++ b/app/models/application_model/checks_attribute_values_and_length.rb @@ -0,0 +1,54 @@ +# Copyright (C) 2012-2016 Zammad Foundation, http://zammad-foundation.org/ +module ApplicationModel::ChecksAttributeValuesAndLength + extend ActiveSupport::Concern + + included do + before_create :check_attribute_values_and_length + before_update :check_attribute_values_and_length + end + +=begin + +1) check string/varchar size and cut them if needed + +2) check string for null byte \u0000 and remove it + +=end + + def check_attribute_values_and_length + columns = self.class.columns_hash + attributes.each { |name, value| + next if value.blank? + next if !value.instance_of?(String) + column = columns[name] + next if !column + + # strip null byte chars (postgresql will complain about it) + if column.sql_type == 'text' + if Rails.application.config.db_null_byte == false + self[name].delete!("\u0000") + end + end + + # for varchar check length and replace null bytes + limit = column.limit + if limit + current_length = value.length + if limit < current_length + logger.warn "WARNING: cut string because of database length #{self.class}.#{name}(#{limit} but is #{current_length}:#{value})" + self[name] = value[0, limit] + end + + # strip null byte chars (postgresql will complain about it) + if Rails.application.config.db_null_byte == false + self[name].delete!("\u0000") + end + end + + # strip 4 bytes utf8 chars if needed (mysql/mariadb will complain it) + next if self[name].blank? + self[name] = self[name].utf8_to_3bytesutf8 + } + true + end +end diff --git a/config/initializers/db_preferences_mysql.rb b/config/initializers/db_preferences_mysql.rb index 121608f10..9de0ac255 100644 --- a/config/initializers/db_preferences_mysql.rb +++ b/config/initializers/db_preferences_mysql.rb @@ -1,6 +1,7 @@ # mysql if ActiveRecord::Base.connection_config[:adapter] == 'mysql2' Rails.application.config.db_4bytes_utf8 = false + Rails.application.config.db_null_byte = true # mysql version check # mysql example: "5.7.3" diff --git a/config/initializers/db_preferences_postgresql.rb b/config/initializers/db_preferences_postgresql.rb index 8f72504c4..b62f7082c 100644 --- a/config/initializers/db_preferences_postgresql.rb +++ b/config/initializers/db_preferences_postgresql.rb @@ -2,6 +2,7 @@ if ActiveRecord::Base.connection_config[:adapter] == 'postgresql' Rails.application.config.db_case_sensitive = true Rails.application.config.db_like = 'ILIKE' + Rails.application.config.db_null_byte = false # postgresql version check # example output: "9.5.0" diff --git a/lib/stats/ticket_reopen.rb b/lib/stats/ticket_reopen.rb index 38b3afacc..360d549d4 100644 --- a/lib/stats/ticket_reopen.rb +++ b/lib/stats/ticket_reopen.rb @@ -66,6 +66,7 @@ class Stats::TicketReopen def self.log(object, o_id, changes, updated_by_id) return if object != 'Ticket' ticket = Ticket.lookup(id: o_id) + return if !ticket # check if close_at is already set / if not, ticket is not reopend return if !ticket.close_at diff --git a/test/unit/ticket_null_byte_test.rb b/test/unit/ticket_null_byte_test.rb new file mode 100644 index 000000000..aa3fdbd6a --- /dev/null +++ b/test/unit/ticket_null_byte_test.rb @@ -0,0 +1,34 @@ +# encoding: utf-8 +require 'test_helper' + +class TicketNullByteTest < ActiveSupport::TestCase + test 'null byte test' do + ticket1 = Ticket.create!( + title: "some title \u0000 123", + group: Group.lookup(name: 'Users'), + customer_id: 2, + updated_by_id: 1, + created_by_id: 1, + ) + assert(ticket1, 'ticket created') + + article1 = Ticket::Article.create!( + ticket_id: ticket1.id, + from: 'some_customer_com-1@example.com', + to: 'some_zammad_com-1@example.com', + subject: "com test 1\u0000", + message_id: 'some@id_com_1', + body: "some\u0000message 123", + internal: false, + sender: Ticket::Article::Sender.find_by(name: 'Customer'), + type: Ticket::Article::Type.find_by(name: 'email'), + updated_by_id: 1, + created_by_id: 1, + ) + assert(article1, 'ticket created') + + ticket1.destroy! + article1.destroy! + + end +end