Improved bot detection - issue #1207. Added ticket limits per hour and day.

This commit is contained in:
Martin Edenhofer 2017-06-28 19:13:52 +02:00
parent 4aa1ec84be
commit 1d4ce42c67
11 changed files with 392 additions and 62 deletions

View file

@ -281,6 +281,7 @@ test:integration:es_mysql:
- ruby -I test/ test/integration/elasticsearch_test.rb
- ruby -I test/ test/controllers/search_controller_test.rb
- ruby -I test/ test/integration/report_test.rb
- ruby -I test/ test/controllers/form_controller_test.rb
- rake db:drop
test:integration:es_postgresql:
@ -297,6 +298,7 @@ test:integration:es_postgresql:
- ruby -I test/ test/integration/elasticsearch_test.rb
- ruby -I test/ test/controllers/search_controller_test.rb
- ruby -I test/ test/integration/report_test.rb
- ruby -I test/ test/controllers/form_controller_test.rb
- rake db:drop
test:integration:zendesk_mysql:

View file

@ -45,7 +45,7 @@ gem 'twitter'
gem 'telegramAPI'
gem 'koala'
gem 'mail'
gem 'email_verifier'
gem 'valid_email2'
gem 'htmlentities'
gem 'mime-types'

View file

@ -93,7 +93,6 @@ GEM
delayed_job (>= 3.0, < 5)
diff-lcs (1.2.5)
diffy (3.1.0)
dnsruby (1.59.3)
docile (1.1.5)
domain_name (0.5.20170404)
unf (>= 0.0.5, < 1.0.0)
@ -107,8 +106,6 @@ GEM
em-websocket (0.5.1)
eventmachine (>= 0.12.9)
http_parser.rb (~> 0.6.0)
email_verifier (0.1.0)
dnsruby (>= 1.5)
equalizer (0.0.10)
erubis (2.7.0)
eventmachine (1.2.3)
@ -403,6 +400,9 @@ GEM
unicorn (5.2.0)
kgio (~> 2.6)
raindrops (~> 0.7)
valid_email2 (1.2.17)
activemodel (>= 3.2)
mail (~> 2.5)
webmock (2.3.2)
addressable (>= 2.3.6)
crack (>= 0.3.2)
@ -439,7 +439,6 @@ DEPENDENCIES
doorkeeper
eco
em-websocket
email_verifier
eventmachine
execjs
factory_girl_rails
@ -491,6 +490,7 @@ DEPENDENCIES
twitter
uglifier
unicorn
valid_email2
webmock
writeexcel
zendesk_api

View file

@ -7,6 +7,8 @@ class FormController < ApplicationController
def config
return if !enabled?
return if !fingerprint_exists?
return if limit_reached?
api_path = Rails.configuration.api_path
http_type = Setting.get('http_type')
@ -17,6 +19,7 @@ class FormController < ApplicationController
config = {
enabled: Setting.get('form_ticket_create'),
endpoint: endpoint,
token: token_gen(params[:fingerprint])
}
if params[:test] && current_user && current_user.permissions?('admin.channel_formular')
@ -28,35 +31,35 @@ class FormController < ApplicationController
def submit
return if !enabled?
return if !fingerprint_exists?
return if !token_valid?(params[:token], params[:fingerprint])
return if limit_reached?
# validate input
errors = {}
if !params[:name] || params[:name].empty?
if params[:name].blank?
errors['name'] = 'required'
end
if !params[:email] || params[:email].empty?
if params[:email].blank?
errors['email'] = 'required'
end
if params[:email] !~ /@/
elsif params[:email] !~ /@/
errors['email'] = 'invalid'
elsif params[:email] =~ /(>|<|\||\!|"|§|'|\$|%|&|\(|\)|\?|\s|\.\.)/
errors['email'] = 'invalid'
end
if params[:email] =~ /(>|<|\||\!|"|§|'|\$|%|&|\(|\)|\?|\s)/
errors['email'] = 'invalid'
end
if !params[:title] || params[:title].empty?
if params[:title].blank?
errors['title'] = 'required'
end
if !params[:body] || params[:body].empty?
if params[:body].blank?
errors['body'] = 'required'
end
# realtime verify
if !errors['email']
if errors['email'].blank?
begin
checker = EmailVerifier::Checker.new(params[:email])
checker.connect
if !checker.verify
errors['email'] = "Unable to send to '#{params[:email]}'"
address = ValidEmail2::Address.new(params[:email])
if !address || !address.valid? || !address.valid_mx?
errors['email'] = 'invalid'
end
rescue => e
message = e.to_s
@ -69,7 +72,7 @@ class FormController < ApplicationController
end
end
if errors && !errors.empty?
if errors.present?
render json: {
errors: errors
}, status: :ok
@ -86,7 +89,6 @@ class FormController < ApplicationController
firstname: name,
lastname: '',
email: email,
password: '',
active: true,
role_ids: role_ids,
updated_by_id: 1,
@ -97,10 +99,20 @@ class FormController < ApplicationController
# set current user
UserInfo.current_user_id = customer.id
group = Group.where(active: true).first
if !group
group = Group.first
end
ticket = Ticket.create!(
group_id: 1,
group_id: group.id,
customer_id: customer.id,
title: params[:title],
preferences: {
form: {
remote_ip: request.remote_ip,
fingerprint_md5: Digest::MD5.hexdigest(params[:fingerprint]),
}
}
)
article = Ticket::Article.create!(
ticket_id: ticket.id,
@ -138,6 +150,91 @@ class FormController < ApplicationController
private
def token_gen(fingerprint)
crypt = ActiveSupport::MessageEncryptor.new(Setting.get('application_secret'))
fingerprint = "#{Base64.strict_encode64(Setting.get('fqdn'))}:#{Time.zone.now.to_i}:#{Base64.strict_encode64(fingerprint)}"
Base64.strict_encode64(crypt.encrypt_and_sign(fingerprint))
end
def token_valid?(token, fingerprint)
if token.blank?
Rails.logger.info 'No token for form!'
response_access_deny
return false
end
begin
crypt = ActiveSupport::MessageEncryptor.new(Setting.get('application_secret'))
result = crypt.decrypt_and_verify(Base64.decode64(token))
rescue
Rails.logger.info 'Invalid token for form!'
response_access_deny
return false
end
if result.blank?
Rails.logger.info 'Invalid token for form!'
response_access_deny
return false
end
parts = result.split(/:/)
if parts.count != 3
Rails.logger.info "Invalid token for form (need to have 3 parts, only #{parts.count} found)!"
response_access_deny
return false
end
fqdn_local = Base64.decode64(parts[0])
if fqdn_local != Setting.get('fqdn')
Rails.logger.info "Invalid token for form (invalid fqdn found #{fqdn_local} != #{Setting.get('fqdn')})!"
response_access_deny
return false
end
fingerprint_local = Base64.decode64(parts[2])
if fingerprint_local != fingerprint
Rails.logger.info "Invalid token for form (invalid fingerprint found #{fingerprint_local} != #{fingerprint})!"
response_access_deny
return false
end
if parts[1].to_i < (Time.zone.now.to_i - 60 * 60 * 24)
Rails.logger.info 'Invalid token for form (token expired})!'
response_access_deny
return false
end
true
end
def limit_reached?
return false if !SearchIndexBackend.enabled?
form_limit_by_ip_per_hour = Setting.get('form_ticket_create_by_ip_per_hour') || 20
result = SearchIndexBackend.search("preferences.form.remote_ip:'#{request.remote_ip}' AND created_at:>now-1h", form_limit_by_ip_per_hour, 'Ticket')
if result.count >= form_limit_by_ip_per_hour.to_i
response_access_deny
return true
end
form_limit_by_ip_per_day = Setting.get('form_ticket_create_by_ip_per_day') || 240
result = SearchIndexBackend.search("preferences.form.remote_ip:'#{request.remote_ip}' AND created_at:>now-1d", form_limit_by_ip_per_day, 'Ticket')
if result.count >= form_limit_by_ip_per_day.to_i
response_access_deny
return true
end
form_limit_per_day = Setting.get('form_ticket_create_per_day') || 5000
result = SearchIndexBackend.search('preferences.form.remote_ip:* AND created_at:>now-1d', form_limit_per_day, 'Ticket')
if result.count >= form_limit_per_day.to_i
response_access_deny
return true
end
false
end
def fingerprint_exists?
return true if params[:fingerprint].present? && params[:fingerprint].length > 30
Rails.logger.info 'No fingerprint given!'
response_access_deny
false
end
def enabled?
return true if params[:test] && current_user && current_user.permissions?('admin.channel_formular')
return true if Setting.get('form_ticket_create')

View file

@ -3,6 +3,6 @@ Zammad::Application.routes.draw do
# forms
match api_path + '/form_submit', to: 'form#submit', via: :post
match api_path + '/form_config', to: 'form#config', via: :get
match api_path + '/form_config', to: 'form#config', via: :post
end

View file

@ -56,7 +56,7 @@ or
def self.email(params)
# send verify email
subject = if !params[:subject] || params[:subject].empty?
subject = if params[:subject].blank?
'#' + rand(99_999_999_999).to_s
else
params[:subject]

View file

@ -427,8 +427,7 @@ return true if backend is configured
=end
def self.enabled?
return if !Setting.get('es_url')
return if Setting.get('es_url').empty?
return false if Setting.get('es_url').blank?
true
end

View file

@ -14,7 +14,7 @@
<div id="feedback-form-inline"></div>
<div class="js-logDisplay"></div>
<div class="js-logDisplay" style="overflow-x: hidden;"></div>
<p>Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat.</p>

View file

@ -204,8 +204,14 @@ $(function() {
if (this.options.test) {
params.test = true
}
params.fingerprint = this.fingerprint()
$.ajax({
method: 'post',
url: _this.endpoint_config,
cache: false,
processData: true,
data: params
}).done(function(data) {
_this.log('debug', 'config:', data)
@ -256,7 +262,7 @@ $(function() {
_this.log('debug', 'currentTime', currentTime)
_this.log('debug', 'modalOpenTime', _this.modalOpenTime.getTime())
_this.log('debug', 'diffTime', diff)
if (diff < 1000*8) {
if (diff < 1000*10) {
alert('Sorry, you look like an robot!')
return
}
@ -317,7 +323,10 @@ $(function() {
formData.append('test', true)
}
formData.append('token', this._config.token)
formData.append('fingerprint', this.fingerprint())
_this.log('debug', 'formData', formData)
return formData
}
@ -463,6 +472,22 @@ $(function() {
return string
}
Plugin.prototype.fingerprint = function () {
var canvas = document.createElement('canvas')
var ctx = canvas.getContext('2d')
var txt = 'https://zammad.com'
ctx.textBaseline = 'top'
ctx.font = '12px \'Arial\''
ctx.textBaseline = 'alphabetic'
ctx.fillStyle = '#f60'
ctx.fillRect(125,1,62,20)
ctx.fillStyle = '#069'
ctx.fillText(txt, 2, 15)
ctx.fillStyle = 'rgba(100, 200, 0, 0.7)'
ctx.fillText(txt, 4, 17)
return canvas.toDataURL()
}
$.fn[pluginName] = function (options) {
return this.each(function () {
var instance = $.data(this, 'plugin_' + pluginName)

View file

@ -82,23 +82,6 @@ class FormTest < TestCase
browser: agent,
css: 'body div.zammad-form-modal button[type="submit"][disabled]',
)
set(
browser: agent,
css: 'body div.zammad-form-modal [name="email"]',
value: 'notexistinginanydomainspacealsonothere@znuny.com',
)
click(
browser: agent,
css: 'body div.zammad-form-modal button[type="submit"]',
)
watch_for(
browser: agent,
css: 'body div.zammad-form-modal .has-error [name="email"]',
)
watch_for_disappear(
browser: agent,
css: 'body div.zammad-form-modal button[type="submit"][disabled]',
)
set(
browser: agent,
css: 'body div.zammad-form-modal [name="email"]',
@ -315,23 +298,6 @@ class FormTest < TestCase
browser: customer,
css: 'body div.zammad-form-modal button[type="submit"][disabled]',
)
set(
browser: customer,
css: 'body div.zammad-form-modal [name="email"]',
value: 'notexistinginanydomainspacealsonothere@znuny.com',
)
click(
browser: customer,
css: 'body div.zammad-form-modal button[type="submit"]',
)
watch_for(
browser: customer,
css: 'body div.zammad-form-modal .has-error [name="email"]',
)
watch_for_disappear(
browser: customer,
css: 'body div.zammad-form-modal button[type="submit"][disabled]',
)
set(
browser: customer,
css: 'body div.zammad-form-modal [name="email"]',

View file

@ -0,0 +1,241 @@
# encoding: utf-8
require 'test_helper'
require 'rake'
class FormControllerTest < ActionDispatch::IntegrationTest
setup do
@headers = { 'ACCEPT' => 'application/json', 'CONTENT_TYPE' => 'application/json', 'REMOTE_ADDR' => '1.2.3.4' }
if ENV['ES_URL'].present?
#fail "ERROR: Need ES_URL - hint ES_URL='http://127.0.0.1:9200'"
Setting.set('es_url', ENV['ES_URL'])
# Setting.set('es_url', 'http://127.0.0.1:9200')
# Setting.set('es_index', 'estest.local_zammad')
# Setting.set('es_user', 'elasticsearch')
# Setting.set('es_password', 'zammad')
if ENV['ES_INDEX_RAND'].present?
ENV['ES_INDEX'] = "es_index_#{rand(999_999_999)}"
end
if ENV['ES_INDEX'].blank?
raise "ERROR: Need ES_INDEX - hint ES_INDEX='estest.local_zammad'"
end
Setting.set('es_index', ENV['ES_INDEX'])
end
Ticket.destroy_all
# drop/create indexes
Setting.reload
Rake::Task.clear
Zammad::Application.load_tasks
Rake::Task['searchindex:rebuild'].execute
end
test '01 - get config call' do
post '/api/v1/form_config', {}.to_json, @headers
assert_response(401)
result = JSON.parse(@response.body)
assert_equal(result.class, Hash)
assert_equal(result['error'], 'Not authorized')
end
test '02 - get config call' do
Setting.set('form_ticket_create', true)
post '/api/v1/form_config', {}.to_json, @headers
assert_response(401)
result = JSON.parse(@response.body)
assert_equal(result.class, Hash)
assert_equal(result['error'], 'Not authorized')
end
test '03 - get config call & do submit' do
Setting.set('form_ticket_create', true)
fingerprint = SecureRandom.hex(40)
post '/api/v1/form_config', { fingerprint: fingerprint }.to_json, @headers
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(result.class, Hash)
assert_equal(result['enabled'], true)
assert_equal(result['endpoint'], 'http://zammad.example.com/api/v1/form_submit')
assert(result['token'])
token = result['token']
post '/api/v1/form_submit', { fingerprint: fingerprint, token: 'invalid' }.to_json, @headers
assert_response(401)
result = JSON.parse(@response.body)
assert_equal(result.class, Hash)
assert_equal(result['error'], 'Not authorized')
post '/api/v1/form_submit', { fingerprint: fingerprint, token: token }.to_json, @headers
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(result.class, Hash)
assert(result['errors'])
assert_equal(result['errors']['name'], 'required')
assert_equal(result['errors']['email'], 'required')
assert_equal(result['errors']['title'], 'required')
assert_equal(result['errors']['body'], 'required')
post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, email: 'some' }.to_json, @headers
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(result.class, Hash)
assert(result['errors'])
assert_equal(result['errors']['name'], 'required')
assert_equal(result['errors']['email'], 'invalid')
assert_equal(result['errors']['title'], 'required')
assert_equal(result['errors']['body'], 'required')
post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, name: 'Bob Smith', email: 'discard@znuny.com', title: 'test', body: 'hello' }.to_json, @headers
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(result.class, Hash)
assert_not(result['errors'])
assert(result['ticket'])
assert(result['ticket']['id'])
assert(result['ticket']['number'])
travel 5.hours
post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, name: 'Bob Smith', email: 'discard@znuny.com', title: 'test', body: 'hello' }.to_json, @headers
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(result.class, Hash)
assert_not(result['errors'])
assert(result['ticket'])
assert(result['ticket']['id'])
assert(result['ticket']['number'])
travel 20.hours
post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, name: 'Bob Smith', email: 'discard@znuny.com', title: 'test', body: 'hello' }.to_json, @headers
assert_response(401)
end
test '04 - get config call & do submit' do
Setting.set('form_ticket_create', true)
fingerprint = SecureRandom.hex(40)
post '/api/v1/form_config', { fingerprint: fingerprint }.to_json, @headers
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(result.class, Hash)
assert_equal(result['enabled'], true)
assert_equal(result['endpoint'], 'http://zammad.example.com/api/v1/form_submit')
assert(result['token'])
token = result['token']
post '/api/v1/form_submit', { fingerprint: fingerprint, token: 'invalid' }.to_json, @headers
assert_response(401)
result = JSON.parse(@response.body)
assert_equal(result.class, Hash)
assert_equal(result['error'], 'Not authorized')
post '/api/v1/form_submit', { fingerprint: fingerprint, token: token }.to_json, @headers
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(result.class, Hash)
assert(result['errors'])
assert_equal(result['errors']['name'], 'required')
assert_equal(result['errors']['email'], 'required')
assert_equal(result['errors']['title'], 'required')
assert_equal(result['errors']['body'], 'required')
post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, email: 'some' }.to_json, @headers
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(result.class, Hash)
assert(result['errors'])
assert_equal(result['errors']['name'], 'required')
assert_equal(result['errors']['email'], 'invalid')
assert_equal(result['errors']['title'], 'required')
assert_equal(result['errors']['body'], 'required')
post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, name: 'Bob Smith', email: 'somebody@example.com', title: 'test', body: 'hello' }.to_json, @headers
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(result.class, Hash)
assert(result['errors'])
assert_equal(result['errors']['email'], 'invalid')
end
test '05 - limits' do
return if !SearchIndexBackend.enabled?
Setting.set('form_ticket_create', true)
fingerprint = SecureRandom.hex(40)
post '/api/v1/form_config', { fingerprint: fingerprint }.to_json, @headers
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(result.class, Hash)
assert_equal(result['enabled'], true)
assert_equal(result['endpoint'], 'http://zammad.example.com/api/v1/form_submit')
assert(result['token'])
token = result['token']
(1..20).each { |count|
travel 10.seconds
post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, name: 'Bob Smith', email: 'discard@znuny.com', title: "test#{count}", body: 'hello' }.to_json, @headers
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(result.class, Hash)
assert_not(result['errors'])
assert(result['ticket'])
assert(result['ticket']['id'])
assert(result['ticket']['number'])
Scheduler.worker(true)
sleep 1 # wait until elasticsearch is index
}
sleep 10 # wait until elasticsearch is index
post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, name: 'Bob Smith', email: 'discard@znuny.com', title: 'test-last', body: 'hello' }.to_json, @headers
assert_response(401)
result = JSON.parse(@response.body)
assert_equal(result.class, Hash)
assert(result['error'])
@headers = { 'ACCEPT' => 'application/json', 'CONTENT_TYPE' => 'application/json', 'REMOTE_ADDR' => '1.2.3.5' }
(1..20).each { |count|
travel 10.seconds
post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, name: 'Bob Smith', email: 'discard@znuny.com', title: "test-2-#{count}", body: 'hello' }.to_json, @headers
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(result.class, Hash)
assert_not(result['errors'])
assert(result['ticket'])
assert(result['ticket']['id'])
assert(result['ticket']['number'])
Scheduler.worker(true)
sleep 1 # wait until elasticsearch is index
}
sleep 10 # wait until elasticsearch is index
post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, name: 'Bob Smith', email: 'discard@znuny.com', title: 'test-2-last', body: 'hello' }.to_json, @headers
assert_response(401)
result = JSON.parse(@response.body)
assert_equal(result.class, Hash)
assert(result['error'])
end
end