Improved bot detection - issue #1207. Added ticket limits per hour and day.

.gitlab-ci.yml

@@ -281,6 +281,7 @@ test:integration:es_mysql:
     - ruby -I test/ test/integration/elasticsearch_test.rb
     - ruby -I test/ test/controllers/search_controller_test.rb
     - ruby -I test/ test/integration/report_test.rb
+    - ruby -I test/ test/controllers/form_controller_test.rb
     - rake db:drop
 
 test:integration:es_postgresql:
@@ -297,6 +298,7 @@ test:integration:es_postgresql:
     - ruby -I test/ test/integration/elasticsearch_test.rb
     - ruby -I test/ test/controllers/search_controller_test.rb
     - ruby -I test/ test/integration/report_test.rb
+    - ruby -I test/ test/controllers/form_controller_test.rb
     - rake db:drop
 
 test:integration:zendesk_mysql:

Gemfile

@@ -45,7 +45,7 @@ gem 'twitter'
 gem 'telegramAPI'
 gem 'koala'
 gem 'mail'
-gem 'email_verifier'
+gem 'valid_email2'
 gem 'htmlentities'
 
 gem 'mime-types'

Gemfile.lock

@@ -93,7 +93,6 @@ GEM
       delayed_job (>= 3.0, < 5)
     diff-lcs (1.2.5)
     diffy (3.1.0)
-    dnsruby (1.59.3)
     docile (1.1.5)
     domain_name (0.5.20170404)
       unf (>= 0.0.5, < 1.0.0)
@@ -107,8 +106,6 @@ GEM
     em-websocket (0.5.1)
       eventmachine (>= 0.12.9)
       http_parser.rb (~> 0.6.0)
-    email_verifier (0.1.0)
-      dnsruby (>= 1.5)
     equalizer (0.0.10)
     erubis (2.7.0)
     eventmachine (1.2.3)
@@ -403,6 +400,9 @@ GEM
     unicorn (5.2.0)
       kgio (~> 2.6)
       raindrops (~> 0.7)
+    valid_email2 (1.2.17)
+      activemodel (>= 3.2)
+      mail (~> 2.5)
     webmock (2.3.2)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
@@ -439,7 +439,6 @@ DEPENDENCIES
   doorkeeper
   eco
   em-websocket
-  email_verifier
   eventmachine
   execjs
   factory_girl_rails
@@ -491,6 +490,7 @@ DEPENDENCIES
   twitter
   uglifier
   unicorn
+  valid_email2
   webmock
   writeexcel
   zendesk_api

app/controllers/form_controller.rb

@@ -7,6 +7,8 @@ class FormController < ApplicationController
 
   def config
     return if !enabled?
+    return if !fingerprint_exists?
+    return if limit_reached?
 
     api_path  = Rails.configuration.api_path
     http_type = Setting.get('http_type')
@@ -17,6 +19,7 @@ class FormController < ApplicationController
     config = {
       enabled:  Setting.get('form_ticket_create'),
       endpoint: endpoint,
+      token:    token_gen(params[:fingerprint])
     }
 
     if params[:test] && current_user && current_user.permissions?('admin.channel_formular')
@@ -28,35 +31,35 @@ class FormController < ApplicationController
 
   def submit
     return if !enabled?
+    return if !fingerprint_exists?
+    return if !token_valid?(params[:token], params[:fingerprint])
+    return if limit_reached?
 
     # validate input
     errors = {}
-    if !params[:name] || params[:name].empty?
+    if params[:name].blank?
       errors['name'] = 'required'
     end
-    if !params[:email] || params[:email].empty?
+    if params[:email].blank?
       errors['email'] = 'required'
-    end
+    elsif params[:email] !~ /@/
-    if params[:email] !~ /@/
+      errors['email'] = 'invalid'
+    elsif params[:email] =~ /(>|<|\||\!|"|§|'|\$|%|&|\(|\)|\?|\s|\.\.)/
       errors['email'] = 'invalid'
     end
-    if params[:email] =~ /(>|<|\||\!|"|§|'|\$|%|&|\(|\)|\?|\s)/
+    if params[:title].blank?
-      errors['email'] = 'invalid'
-    end
-    if !params[:title] || params[:title].empty?
       errors['title'] = 'required'
     end
-    if !params[:body] || params[:body].empty?
+    if params[:body].blank?
       errors['body'] = 'required'
     end
 
     # realtime verify
-    if !errors['email']
+    if errors['email'].blank?
       begin
-        checker = EmailVerifier::Checker.new(params[:email])
+        address = ValidEmail2::Address.new(params[:email])
-        checker.connect
+        if !address || !address.valid? || !address.valid_mx?
-        if !checker.verify
+          errors['email'] = 'invalid'
-          errors['email'] = "Unable to send to '#{params[:email]}'"
         end
       rescue => e
         message = e.to_s
@@ -69,7 +72,7 @@ class FormController < ApplicationController
       end
     end
 
-    if errors && !errors.empty?
+    if errors.present?
       render json: {
         errors: errors
       }, status: :ok
@@ -86,7 +89,6 @@ class FormController < ApplicationController
         firstname: name,
         lastname: '',
         email: email,
-        password: '',
         active: true,
         role_ids: role_ids,
         updated_by_id: 1,
@@ -97,10 +99,20 @@ class FormController < ApplicationController
     # set current user
     UserInfo.current_user_id = customer.id
 
+    group = Group.where(active: true).first
+    if !group
+      group = Group.first
+    end
     ticket = Ticket.create!(
-      group_id: 1,
+      group_id: group.id,
       customer_id: customer.id,
       title: params[:title],
+      preferences: {
+        form: {
+          remote_ip: request.remote_ip,
+          fingerprint_md5: Digest::MD5.hexdigest(params[:fingerprint]),
+        }
+      }
     )
     article = Ticket::Article.create!(
       ticket_id: ticket.id,
@@ -138,6 +150,91 @@ class FormController < ApplicationController
 
   private
 
+  def token_gen(fingerprint)
+    crypt = ActiveSupport::MessageEncryptor.new(Setting.get('application_secret'))
+    fingerprint = "#{Base64.strict_encode64(Setting.get('fqdn'))}:#{Time.zone.now.to_i}:#{Base64.strict_encode64(fingerprint)}"
+    Base64.strict_encode64(crypt.encrypt_and_sign(fingerprint))
+  end
+
+  def token_valid?(token, fingerprint)
+    if token.blank?
+      Rails.logger.info 'No token for form!'
+      response_access_deny
+      return false
+    end
+    begin
+      crypt = ActiveSupport::MessageEncryptor.new(Setting.get('application_secret'))
+      result = crypt.decrypt_and_verify(Base64.decode64(token))
+    rescue
+      Rails.logger.info 'Invalid token for form!'
+      response_access_deny
+      return false
+    end
+    if result.blank?
+      Rails.logger.info 'Invalid token for form!'
+      response_access_deny
+      return false
+    end
+    parts = result.split(/:/)
+    if parts.count != 3
+      Rails.logger.info "Invalid token for form (need to have 3 parts, only #{parts.count} found)!"
+      response_access_deny
+      return false
+    end
+    fqdn_local = Base64.decode64(parts[0])
+    if fqdn_local != Setting.get('fqdn')
+      Rails.logger.info "Invalid token for form (invalid fqdn found #{fqdn_local} != #{Setting.get('fqdn')})!"
+      response_access_deny
+      return false
+    end
+    fingerprint_local = Base64.decode64(parts[2])
+    if fingerprint_local != fingerprint
+      Rails.logger.info "Invalid token for form (invalid fingerprint found #{fingerprint_local} != #{fingerprint})!"
+      response_access_deny
+      return false
+    end
+    if parts[1].to_i < (Time.zone.now.to_i - 60 * 60 * 24)
+      Rails.logger.info 'Invalid token for form (token expired})!'
+      response_access_deny
+      return false
+    end
+    true
+  end
+
+  def limit_reached?
+    return false if !SearchIndexBackend.enabled?
+
+    form_limit_by_ip_per_hour = Setting.get('form_ticket_create_by_ip_per_hour') || 20
+    result = SearchIndexBackend.search("preferences.form.remote_ip:'#{request.remote_ip}' AND created_at:>now-1h", form_limit_by_ip_per_hour, 'Ticket')
+    if result.count >= form_limit_by_ip_per_hour.to_i
+      response_access_deny
+      return true
+    end
+
+    form_limit_by_ip_per_day = Setting.get('form_ticket_create_by_ip_per_day') || 240
+    result = SearchIndexBackend.search("preferences.form.remote_ip:'#{request.remote_ip}' AND created_at:>now-1d", form_limit_by_ip_per_day, 'Ticket')
+    if result.count >= form_limit_by_ip_per_day.to_i
+      response_access_deny
+      return true
+    end
+
+    form_limit_per_day = Setting.get('form_ticket_create_per_day') || 5000
+    result = SearchIndexBackend.search('preferences.form.remote_ip:* AND created_at:>now-1d', form_limit_per_day, 'Ticket')
+    if result.count >= form_limit_per_day.to_i
+      response_access_deny
+      return true
+    end
+
+    false
+  end
+
+  def fingerprint_exists?
+    return true if params[:fingerprint].present? && params[:fingerprint].length > 30
+    Rails.logger.info 'No fingerprint given!'
+    response_access_deny
+    false
+  end
+
   def enabled?
     return true if params[:test] && current_user && current_user.permissions?('admin.channel_formular')
     return true if Setting.get('form_ticket_create')

config/routes/form.rb

@@ -3,6 +3,6 @@ Zammad::Application.routes.draw do
 
   # forms
   match api_path + '/form_submit',      to: 'form#submit',    via: :post
-  match api_path + '/form_config',      to: 'form#config',    via: :get
+  match api_path + '/form_config',      to: 'form#config',    via: :post
 
 end

lib/email_helper/verify.rb

@@ -56,7 +56,7 @@ or
     def self.email(params)
 
       # send verify email
-      subject = if !params[:subject] || params[:subject].empty?
+      subject = if params[:subject].blank?
                   '#' + rand(99_999_999_999).to_s
                 else
                   params[:subject]

lib/search_index_backend.rb

@@ -427,8 +427,7 @@ return true if backend is configured
 =end
 
   def self.enabled?
-    return if !Setting.get('es_url')
+    return false if Setting.get('es_url').blank?
-    return if Setting.get('es_url').empty?
     true
   end
 

public/assets/form/form.html

@@ -14,7 +14,7 @@
 <div id="feedback-form-inline"></div>
 
 
-<div class="js-logDisplay"></div>
+<div class="js-logDisplay" style="overflow-x: hidden;"></div>
 
 <p>Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat.</p>
 

public/assets/form/form.js

@@ -204,8 +204,14 @@ $(function() {
     if (this.options.test) {
       params.test = true
     }
+
+    params.fingerprint = this.fingerprint()
+
     $.ajax({
+      method: 'post',
       url: _this.endpoint_config,
+      cache: false,
+      processData: true,
       data: params
     }).done(function(data) {
       _this.log('debug', 'config:', data)
@@ -256,7 +262,7 @@ $(function() {
       _this.log('debug', 'currentTime', currentTime)
       _this.log('debug', 'modalOpenTime', _this.modalOpenTime.getTime())
       _this.log('debug', 'diffTime', diff)
-      if (diff < 1000*8) {
+      if (diff < 1000*10) {
         alert('Sorry, you look like an robot!')
         return
       }
@@ -317,7 +323,10 @@ $(function() {
       formData.append('test', true)
     }
     formData.append('token', this._config.token)
+
+    formData.append('fingerprint', this.fingerprint())
     _this.log('debug', 'formData', formData)
+
     return formData
   }
 
@@ -463,6 +472,22 @@ $(function() {
     return string
   }
 
+  Plugin.prototype.fingerprint = function () {
+    var canvas = document.createElement('canvas')
+    var ctx = canvas.getContext('2d')
+    var txt = 'https://zammad.com'
+    ctx.textBaseline = 'top'
+    ctx.font = '12px \'Arial\''
+    ctx.textBaseline = 'alphabetic'
+    ctx.fillStyle = '#f60'
+    ctx.fillRect(125,1,62,20)
+    ctx.fillStyle = '#069'
+    ctx.fillText(txt, 2, 15)
+    ctx.fillStyle = 'rgba(100, 200, 0, 0.7)'
+    ctx.fillText(txt, 4, 17)
+    return canvas.toDataURL()
+  }
+
   $.fn[pluginName] = function (options) {
     return this.each(function () {
       var instance = $.data(this, 'plugin_' + pluginName)

test/browser/form_test.rb

@@ -82,23 +82,6 @@ class FormTest < TestCase
       browser: agent,
       css: 'body div.zammad-form-modal button[type="submit"][disabled]',
     )
-    set(
-      browser: agent,
-      css: 'body div.zammad-form-modal [name="email"]',
-      value: 'notexistinginanydomainspacealsonothere@znuny.com',
-    )
-    click(
-      browser: agent,
-      css: 'body div.zammad-form-modal button[type="submit"]',
-    )
-    watch_for(
-      browser: agent,
-      css: 'body div.zammad-form-modal .has-error [name="email"]',
-    )
-    watch_for_disappear(
-      browser: agent,
-      css: 'body div.zammad-form-modal button[type="submit"][disabled]',
-    )
     set(
       browser: agent,
       css: 'body div.zammad-form-modal [name="email"]',
@@ -315,23 +298,6 @@ class FormTest < TestCase
       browser: customer,
       css: 'body div.zammad-form-modal button[type="submit"][disabled]',
     )
-    set(
-      browser: customer,
-      css: 'body div.zammad-form-modal [name="email"]',
-      value: 'notexistinginanydomainspacealsonothere@znuny.com',
-    )
-    click(
-      browser: customer,
-      css: 'body div.zammad-form-modal button[type="submit"]',
-    )
-    watch_for(
-      browser: customer,
-      css: 'body div.zammad-form-modal .has-error [name="email"]',
-    )
-    watch_for_disappear(
-      browser: customer,
-      css: 'body div.zammad-form-modal button[type="submit"][disabled]',
-    )
     set(
       browser: customer,
       css: 'body div.zammad-form-modal [name="email"]',

test/controllers/form_controller_test.rb

@@ -0,0 +1,241 @@
+# encoding: utf-8
+require 'test_helper'
+require 'rake'
+
+class FormControllerTest < ActionDispatch::IntegrationTest
+  setup do
+    @headers = { 'ACCEPT' => 'application/json', 'CONTENT_TYPE' => 'application/json', 'REMOTE_ADDR' => '1.2.3.4' }
+
+    if ENV['ES_URL'].present?
+
+      #fail "ERROR: Need ES_URL - hint ES_URL='http://127.0.0.1:9200'"
+      Setting.set('es_url', ENV['ES_URL'])
+
+      # Setting.set('es_url', 'http://127.0.0.1:9200')
+      # Setting.set('es_index', 'estest.local_zammad')
+      # Setting.set('es_user', 'elasticsearch')
+      # Setting.set('es_password', 'zammad')
+
+      if ENV['ES_INDEX_RAND'].present?
+        ENV['ES_INDEX'] = "es_index_#{rand(999_999_999)}"
+      end
+      if ENV['ES_INDEX'].blank?
+        raise "ERROR: Need ES_INDEX - hint ES_INDEX='estest.local_zammad'"
+      end
+      Setting.set('es_index', ENV['ES_INDEX'])
+    end
+
+    Ticket.destroy_all
+
+    # drop/create indexes
+    Setting.reload
+    Rake::Task.clear
+    Zammad::Application.load_tasks
+    Rake::Task['searchindex:rebuild'].execute
+  end
+
+  test '01 - get config call' do
+    post '/api/v1/form_config', {}.to_json, @headers
+    assert_response(401)
+    result = JSON.parse(@response.body)
+    assert_equal(result.class, Hash)
+    assert_equal(result['error'], 'Not authorized')
+  end
+
+  test '02 - get config call' do
+    Setting.set('form_ticket_create', true)
+    post '/api/v1/form_config', {}.to_json, @headers
+    assert_response(401)
+    result = JSON.parse(@response.body)
+    assert_equal(result.class, Hash)
+    assert_equal(result['error'], 'Not authorized')
+
+  end
+
+  test '03 - get config call & do submit' do
+    Setting.set('form_ticket_create', true)
+    fingerprint = SecureRandom.hex(40)
+    post '/api/v1/form_config', { fingerprint: fingerprint }.to_json, @headers
+
+    assert_response(200)
+    result = JSON.parse(@response.body)
+    assert_equal(result.class, Hash)
+    assert_equal(result['enabled'], true)
+    assert_equal(result['endpoint'], 'http://zammad.example.com/api/v1/form_submit')
+    assert(result['token'])
+    token = result['token']
+
+    post '/api/v1/form_submit', { fingerprint: fingerprint, token: 'invalid' }.to_json, @headers
+    assert_response(401)
+    result = JSON.parse(@response.body)
+    assert_equal(result.class, Hash)
+    assert_equal(result['error'], 'Not authorized')
+
+    post '/api/v1/form_submit', { fingerprint: fingerprint, token: token }.to_json, @headers
+    assert_response(200)
+    result = JSON.parse(@response.body)
+    assert_equal(result.class, Hash)
+
+    assert(result['errors'])
+    assert_equal(result['errors']['name'], 'required')
+    assert_equal(result['errors']['email'], 'required')
+    assert_equal(result['errors']['title'], 'required')
+    assert_equal(result['errors']['body'], 'required')
+
+    post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, email: 'some' }.to_json, @headers
+    assert_response(200)
+    result = JSON.parse(@response.body)
+    assert_equal(result.class, Hash)
+
+    assert(result['errors'])
+    assert_equal(result['errors']['name'], 'required')
+    assert_equal(result['errors']['email'], 'invalid')
+    assert_equal(result['errors']['title'], 'required')
+    assert_equal(result['errors']['body'], 'required')
+
+    post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, name: 'Bob Smith', email: 'discard@znuny.com', title: 'test', body: 'hello' }.to_json, @headers
+    assert_response(200)
+    result = JSON.parse(@response.body)
+    assert_equal(result.class, Hash)
+
+    assert_not(result['errors'])
+    assert(result['ticket'])
+    assert(result['ticket']['id'])
+    assert(result['ticket']['number'])
+
+    travel 5.hours
+
+    post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, name: 'Bob Smith', email: 'discard@znuny.com', title: 'test', body: 'hello' }.to_json, @headers
+
+    assert_response(200)
+    result = JSON.parse(@response.body)
+    assert_equal(result.class, Hash)
+
+    assert_not(result['errors'])
+    assert(result['ticket'])
+    assert(result['ticket']['id'])
+    assert(result['ticket']['number'])
+
+    travel 20.hours
+
+    post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, name: 'Bob Smith', email: 'discard@znuny.com', title: 'test', body: 'hello' }.to_json, @headers
+    assert_response(401)
+
+  end
+
+  test '04 - get config call & do submit' do
+    Setting.set('form_ticket_create', true)
+    fingerprint = SecureRandom.hex(40)
+    post '/api/v1/form_config', { fingerprint: fingerprint }.to_json, @headers
+
+    assert_response(200)
+    result = JSON.parse(@response.body)
+    assert_equal(result.class, Hash)
+    assert_equal(result['enabled'], true)
+    assert_equal(result['endpoint'], 'http://zammad.example.com/api/v1/form_submit')
+    assert(result['token'])
+    token = result['token']
+
+    post '/api/v1/form_submit', { fingerprint: fingerprint, token: 'invalid' }.to_json, @headers
+    assert_response(401)
+    result = JSON.parse(@response.body)
+    assert_equal(result.class, Hash)
+    assert_equal(result['error'], 'Not authorized')
+
+    post '/api/v1/form_submit', { fingerprint: fingerprint, token: token }.to_json, @headers
+    assert_response(200)
+    result = JSON.parse(@response.body)
+    assert_equal(result.class, Hash)
+
+    assert(result['errors'])
+    assert_equal(result['errors']['name'], 'required')
+    assert_equal(result['errors']['email'], 'required')
+    assert_equal(result['errors']['title'], 'required')
+    assert_equal(result['errors']['body'], 'required')
+
+    post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, email: 'some' }.to_json, @headers
+    assert_response(200)
+    result = JSON.parse(@response.body)
+    assert_equal(result.class, Hash)
+
+    assert(result['errors'])
+    assert_equal(result['errors']['name'], 'required')
+    assert_equal(result['errors']['email'], 'invalid')
+    assert_equal(result['errors']['title'], 'required')
+    assert_equal(result['errors']['body'], 'required')
+
+    post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, name: 'Bob Smith', email: 'somebody@example.com', title: 'test', body: 'hello' }.to_json, @headers
+    assert_response(200)
+    result = JSON.parse(@response.body)
+    assert_equal(result.class, Hash)
+
+    assert(result['errors'])
+    assert_equal(result['errors']['email'], 'invalid')
+
+  end
+
+  test '05 - limits' do
+    return if !SearchIndexBackend.enabled?
+
+    Setting.set('form_ticket_create', true)
+    fingerprint = SecureRandom.hex(40)
+    post '/api/v1/form_config', { fingerprint: fingerprint }.to_json, @headers
+
+    assert_response(200)
+    result = JSON.parse(@response.body)
+    assert_equal(result.class, Hash)
+    assert_equal(result['enabled'], true)
+    assert_equal(result['endpoint'], 'http://zammad.example.com/api/v1/form_submit')
+    assert(result['token'])
+    token = result['token']
+
+    (1..20).each { |count|
+      travel 10.seconds
+      post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, name: 'Bob Smith', email: 'discard@znuny.com', title: "test#{count}", body: 'hello' }.to_json, @headers
+      assert_response(200)
+      result = JSON.parse(@response.body)
+      assert_equal(result.class, Hash)
+
+      assert_not(result['errors'])
+      assert(result['ticket'])
+      assert(result['ticket']['id'])
+      assert(result['ticket']['number'])
+      Scheduler.worker(true)
+      sleep 1 # wait until elasticsearch is index
+    }
+
+    sleep 10 # wait until elasticsearch is index
+
+    post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, name: 'Bob Smith', email: 'discard@znuny.com', title: 'test-last', body: 'hello' }.to_json, @headers
+    assert_response(401)
+    result = JSON.parse(@response.body)
+    assert_equal(result.class, Hash)
+    assert(result['error'])
+
+    @headers = { 'ACCEPT' => 'application/json', 'CONTENT_TYPE' => 'application/json', 'REMOTE_ADDR' => '1.2.3.5' }
+
+    (1..20).each { |count|
+      travel 10.seconds
+      post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, name: 'Bob Smith', email: 'discard@znuny.com', title: "test-2-#{count}", body: 'hello' }.to_json, @headers
+      assert_response(200)
+      result = JSON.parse(@response.body)
+      assert_equal(result.class, Hash)
+
+      assert_not(result['errors'])
+      assert(result['ticket'])
+      assert(result['ticket']['id'])
+      assert(result['ticket']['number'])
+      Scheduler.worker(true)
+      sleep 1 # wait until elasticsearch is index
+    }
+
+    sleep 10 # wait until elasticsearch is index
+
+    post '/api/v1/form_submit', { fingerprint: fingerprint, token: token, name: 'Bob Smith', email: 'discard@znuny.com', title: 'test-2-last', body: 'hello' }.to_json, @headers
+    assert_response(401)
+    result = JSON.parse(@response.body)
+    assert_equal(result.class, Hash)
+    assert(result['error'])
+  end
+
+end