Fixed issue #1211 - LDAP users that are lost don't get reflected into Zammad.

app/assets/javascripts/app/controllers/_integration/ldap.coffee

@@ -124,7 +124,7 @@ class Form extends App.Controller
     if _.isEmpty(job)
       @lastImport.html('')
       return
-    countDone = job.result.created + job.result.updated + job.result.unchanged + job.result.skipped + job.result.failed
+    countDone = job.result.created + job.result.updated + job.result.unchanged + job.result.skipped + job.result.failed + job.result.deactivated
     if !job.result.roles
       job.result.roles = {}
     for role_id, statistic of job.result.role_ids
@@ -545,6 +545,8 @@ class ConnectionWizard extends App.WizardModal
             total += job.result.unchanged
           if job.result.updated
             total += job.result.updated
+          if job.result.deactivated
+            total += job.result.deactivated
           @$('.js-progress progress').attr('value', total)
           @$('.js-progress progress').attr('max', job.result.sum)
         if job.finished_at
@@ -566,7 +568,7 @@ class ConnectionWizard extends App.WizardModal
     for role_id, statistic of job.result.role_ids
       role = App.Role.find(role_id)
       job.result.roles[role.displayName()] = statistic
-    countDone = job.result.created + job.result.updated + job.result.unchanged + job.result.skipped
+    countDone = job.result.created + job.result.updated + job.result.unchanged + job.result.skipped + job.result.deactivated
     @showSlide('js-try')
     el = $(App.view('integration/ldap_summary')(job: job, countDone: countDone))
     @el.find('.js-summary').html(el)

app/assets/javascripts/app/views/integration/ldap_last_import.jst.eco

@@ -33,13 +33,13 @@
       <ul>
         <li><%- @T('%s user to %s user', 'LDAP', 'Zammad') %> (<%= @countDone %>/<%= @job.result.sum %>):
           <ul>
-            <li><%- @T('Users') %>: <%= @job.result.created %> <%- @T('created') %>, <%= @job.result.updated %> <%- @T('updated') %>, <%= @job.result.unchanged %> <%- @T('untouched') %>, <%= @job.result.skipped %> <%- @T('skipped') %>, <%= @job.result.failed %> <%- @T('failed') %>
+            <li><%- @T('Users') %>: <%= @job.result.created %> <%- @T('created') %>, <%= @job.result.updated %> <%- @T('updated') %>, <%= @job.result.unchanged %> <%- @T('untouched') %>, <%= @job.result.skipped %> <%- @T('skipped') %>, <%= @job.result.failed %> <%- @T('failed') %>, <%= @job.result.deactivated %> <%- @T('deactivated') %>
           </ul>
         <% if !_.isEmpty(@job.result.roles): %>
           <li><%- @T('%s groups to %s roles assignments', 'LDAP', 'Zammad') %>:
             <ul>
               <% for role, result of @job.result.roles: %>
-                <li> <%- @T(role) %>: <%= result.created %> <%- @T('created') %>, <%= result.updated %> <%- @T('updated') %>, <%= result.unchanged %> <%- @T('untouched') %>, <%= result.failed %> <%- @T('failed') %>
+                <li> <%- @T(role) %>: <%= result.created %> <%- @T('created') %>, <%= result.updated %> <%- @T('updated') %>, <%= result.unchanged %> <%- @T('untouched') %>, <%= result.failed %> <%- @T('failed') %>, <%= result.deactivated %> <%- @T('deactivated') %>
               <% end %>
             </ul>
         <% end %>

app/assets/javascripts/app/views/integration/ldap_summary.jst.eco

@@ -1,14 +1,14 @@
 <ul>
   <li><%- @T('%s user to %s user', 'LDAP', 'Zammad') %> (<%= @countDone %>):
     <ul>
-      <li><%- @T('Users') %>: <%= @job.result.created %> <%- @T('created') %>, <%= @job.result.updated %> <%- @T('updated') %>, <%= @job.result.unchanged %> <%- @T('untouched') %>, <%= @job.result.skipped %> <%- @T('skipped') %>, <%= @job.result.failed %> <%- @T('failed') %>
+      <li><%- @T('Users') %>: <%= @job.result.created %> <%- @T('created') %>, <%= @job.result.updated %> <%- @T('updated') %>, <%= @job.result.unchanged %> <%- @T('untouched') %>, <%= @job.result.skipped %> <%- @T('skipped') %>, <%= @job.result.failed %> <%- @T('failed') %>, <%= @job.result.deactivated %> <%- @T('deactivated') %>
     </ul>
   </li>
   <% if !_.isEmpty(@job.result.roles): %>
     <li><%- @T('%s groups to %s roles assignments', 'LDAP', 'Zammad') %>:
       <ul>
         <% for role, result of @job.result.roles: %>
-          <li><%- @T(role) %>: <%= result.created %> <%- @T('created') %>, <%= result.updated %> <%- @T('updated') %>, <%= result.unchanged %> <%- @T('untouched') %>, <%= result.failed %> <%- @T('failed') %>
+          <li><%- @T(role) %>: <%= result.created %> <%- @T('created') %>, <%= result.updated %> <%- @T('updated') %>, <%= result.unchanged %> <%- @T('untouched') %>, <%= result.failed %> <%- @T('failed') %>, <%= result.deactivated %> <%- @T('deactivated') %>
         <% end %>
       </ul>
     </li>

lib/import/base_resource.rb

@@ -16,7 +16,7 @@ module Import
     end
 
     def source
-      import_class_namespace
+      self.class.source
     end
 
     def remote_id(resource, *_args)
@@ -57,6 +57,14 @@ module Import
       changes
     end
 
+    def self.source
+      import_class_namespace
+    end
+
+    def self.import_class_namespace
+      @import_class_namespace ||= name.to_s.sub('Import::', '')
+    end
+
     private
 
     def initialize_associations_states
@@ -214,11 +222,7 @@ module Import
     end
 
     def mapping_config(*_args)
-      import_class_namespace.gsub('::', '_').underscore + '_mapping'
-    end
-
-    def import_class_namespace
-      self.class.name.to_s.sub('Import::', '')
+      self.class.import_class_namespace.gsub('::', '_').underscore + '_mapping'
     end
 
     def handle_args(_resource, *args)

lib/import/ldap/user.rb

@@ -6,6 +6,35 @@ module Import
         @remote_id
       end
 
+      def self.lost_ids(found_remote_ids)
+        ExternalSync.joins('INNER JOIN users ON (users.id = external_syncs.o_id)')
+                    .where(
+                      source: source,
+                      object: import_class.name,
+                      users:  {
+                        active: true
+                      }
+                    )
+                    .pluck(:source_id, :o_id)
+                    .to_h
+                    .except(*found_remote_ids)
+                    .values
+      end
+
+      def self.deactivate_lost(lost_ids)
+        # we need to update in slices since some DBs
+        # have a limit for IN length
+        lost_ids.each_slice(5000) do |slice|
+
+          # we need to instanciate every entry and set
+          # the active state this way to send notifications
+          # to the client
+          ::User.where(id: slice).each do |user|
+            user.update_attribute(:active, false)
+          end
+        end
+      end
+
       private
 
       def import(resource, *args)
@@ -58,7 +87,7 @@ module Import
         return true if resource[:login].blank?
 
         # skip resource if only ignored attributes are set
-        ignored_attributes = %i(login dn created_by_id updated_by_id)
+        ignored_attributes = %i(login dn created_by_id updated_by_id active)
         !resource.except(*ignored_attributes).values.any?(&:present?)
       end
 
@@ -181,6 +210,11 @@ module Import
           mapped[attribute] = mapped[attribute].downcase
         end
 
+        # we have to add the active state manually
+        # because otherwise disabled instances won't get
+        # re-activated if they should get synced again
+        mapped[:active] = true
+
         mapped
       end
 

lib/import/ldap/user_factory.rb

@@ -33,10 +33,13 @@ module Import
         relevant_attributes = config[:user_attributes].keys
         relevant_attributes.push('dn')
 
+        @found_remote_ids = []
         @ldap.search(config[:user_filter], attributes: relevant_attributes) do |entry|
           backend_instance = create_instance(entry, config, user_roles, signup_role_ids, kargs)
           post_import_hook(entry, backend_instance, config, user_roles, signup_role_ids, kargs)
 
+          track_found_remote_ids(backend_instance)
+
           next if import_job.blank?
           import_job_count += 1
           next if import_job_count < 100
@@ -47,6 +50,7 @@ module Import
           import_job_count = 0
         end
 
+        handle_lost
       end
 
       def self.pre_import_hook(_records, *_args)
@@ -77,18 +81,25 @@ module Import
 
         action = backend_instance.action
 
+        add_resource_role_ids_to_statistics(resource.role_ids, action)
+
+        action
+      end
+
+      def self.add_resource_role_ids_to_statistics(role_ids, action)
+        return if role_ids.blank?
+
         known_actions = {
           created:     0,
           updated:     0,
           unchanged:   0,
           failed:      0,
+          deactivated: 0,
         }
 
-        if !@statistics[:role_ids]
-          @statistics[:role_ids] = {}
-        end
+        @statistics[:role_ids] ||= {}
 
-        resource.role_ids.each do |role_id|
+        role_ids.each do |role_id|
 
           next if !known_actions.key?(action)
 
@@ -99,8 +110,6 @@ module Import
 
           @statistics[:role_ids][role_id][action] += 1
         end
-
-        action
       end
 
       def self.user_roles(ldap:, config:)
@@ -111,6 +120,37 @@ module Import
         ldap_group = ::Ldap::Group.new(group_config, ldap: ldap)
         ldap_group.user_roles(config[:group_role_map])
       end
+
+      def self.track_found_remote_ids(backend_instance)
+        @deactivation_actions ||= %i(skipped failed)
+        return if @deactivation_actions.include?(backend_instance.action)
+
+        @found_remote_ids.push(backend_instance.remote_id(nil))
+      end
+
+      def self.handle_lost
+        backend_class = backend_class(nil)
+        lost_ids      = backend_class.lost_ids(@found_remote_ids)
+
+        # track disabled count and substract it from
+        # skipped where they are logged till now
+        @statistics[:deactivated] = lost_ids.size
+        @statistics[:skipped]    -= lost_ids.size
+
+        # loop over every lost user ID and add the
+        # deactivated count to the statistics
+        lost_ids.each do |user_id|
+          role_ids = ::User.joins(:roles)
+                           .where(id: user_id)
+                           .pluck(:'roles_users.role_id')
+
+          add_resource_role_ids_to_statistics(role_ids, :deactivated)
+        end
+
+        # deactivate entries only on live syncs
+        return if @dry_run
+        backend_class.deactivate_lost(lost_ids)
+      end
     end
   end
 end

lib/import/model_resource.rb

@@ -2,11 +2,19 @@ module Import
   class ModelResource < Import::BaseResource
 
     def import_class
-      model_name.constantize
+      self.class.import_class
     end
 
     def model_name
-      @model_name ||= self.class.name.split('::').last
+      self.class.model_name
+    end
+
+    def self.import_class
+      model_name.constantize
+    end
+
+    def self.model_name
+      @model_name ||= name.split('::').last
     end
 
     private

lib/import/statistical_factory.rb

@@ -18,6 +18,7 @@ module Import
         updated:     0,
         unchanged:   0,
         failed:      0,
+        deactivated: 0,
       }
     end
 

spec/lib/import/ldap/user_factory_spec.rb

@@ -49,7 +49,195 @@ RSpec.describe Import::Ldap::UserFactory do
       }.by(1)
     end
 
-    it 'supports dry run' do
+    it 'deactivates lost users' do
+
+      config = {
+        user_filter:     '(objectClass=user)',
+        group_filter:    '(objectClass=group)',
+        user_uid:        'uid',
+        user_attributes: {
+          'uid' => 'login',
+          'email' => 'email',
+        }
+      }
+
+      persistent_entry          = build(:ldap_entry)
+      persistent_entry['uid']   = ['exampleuid']
+      persistent_entry['email'] = ['example@example.com']
+
+      lost_entry          = build(:ldap_entry)
+      lost_entry['uid']   = ['exampleuid_lost']
+      lost_entry['email'] = ['lost@example.com']
+
+      mocked_ldap = double(
+        host:    'ldap.example.com',
+        port:    636,
+        ssl:     true,
+        base_dn: 'dc=example,dc=com'
+      )
+
+      # group user role mapping
+      expect(mocked_ldap).to receive(:search)
+      # user counting
+      expect(mocked_ldap).to receive(:count).and_return(2)
+      # user search
+      expect(mocked_ldap).to receive(:search).and_yield(persistent_entry).and_yield(lost_entry)
+
+      described_class.import(
+        config: config,
+        ldap:   mocked_ldap,
+      )
+
+      # group user role mapping
+      expect(mocked_ldap).to receive(:search)
+      # user counting
+      expect(mocked_ldap).to receive(:count).and_return(1)
+      # user search
+      expect(mocked_ldap).to receive(:search).and_yield(persistent_entry)
+
+      expect do
+        described_class.import(
+          config: config,
+          ldap:   mocked_ldap,
+        )
+      end.to change {
+        User.find_by(email: 'lost@example.com').active
+      }
+    end
+
+    it 're-activates previously lost users' do
+
+      config = {
+        user_filter:     '(objectClass=user)',
+        group_filter:    '(objectClass=group)',
+        user_uid:        'uid',
+        user_attributes: {
+          'uid' => 'login',
+          'email' => 'email',
+        }
+      }
+
+      persistent_entry          = build(:ldap_entry)
+      persistent_entry['uid']   = ['exampleuid']
+      persistent_entry['email'] = ['example@example.com']
+
+      lost_entry          = build(:ldap_entry)
+      lost_entry['uid']   = ['exampleuid_lost']
+      lost_entry['email'] = ['lost@example.com']
+
+      mocked_ldap = double(
+        host:    'ldap.example.com',
+        port:    636,
+        ssl:     true,
+        base_dn: 'dc=example,dc=com'
+      )
+
+      # group user role mapping
+      expect(mocked_ldap).to receive(:search)
+      # user counting
+      expect(mocked_ldap).to receive(:count).and_return(2)
+      # user search
+      expect(mocked_ldap).to receive(:search).and_yield(persistent_entry).and_yield(lost_entry)
+
+      described_class.import(
+        config: config,
+        ldap:   mocked_ldap,
+      )
+
+      # group user role mapping
+      expect(mocked_ldap).to receive(:search)
+      # user counting
+      expect(mocked_ldap).to receive(:count).and_return(1)
+      # user search
+      expect(mocked_ldap).to receive(:search).and_yield(persistent_entry)
+
+      described_class.import(
+        config: config,
+        ldap:   mocked_ldap,
+      )
+
+      # group user role mapping
+      expect(mocked_ldap).to receive(:search)
+      # user counting
+      expect(mocked_ldap).to receive(:count).and_return(2)
+      # user search
+      expect(mocked_ldap).to receive(:search).and_yield(persistent_entry).and_yield(lost_entry)
+
+      expect do
+        described_class.import(
+          config: config,
+          ldap:   mocked_ldap,
+        )
+      end.to change {
+        User.find_by(email: 'lost@example.com').active
+      }
+    end
+
+    it 'deactivates skipped users' do
+
+      config = {
+        user_filter:     '(objectClass=user)',
+        group_filter:    '(objectClass=group)',
+        user_uid:        'uid',
+        user_attributes: {
+          'uid' => 'login',
+          'email' => 'email',
+        },
+      }
+
+      lost_entry          = build(:ldap_entry)
+      lost_entry['uid']   = ['exampleuid']
+      lost_entry['email'] = ['example@example.com']
+
+      mocked_ldap = double(
+        host:    'ldap.example.com',
+        port:    636,
+        ssl:     true,
+        base_dn: 'dc=example,dc=com'
+      )
+
+      # group user role mapping
+      expect(mocked_ldap).to receive(:search)
+      # user counting
+      expect(mocked_ldap).to receive(:count).and_return(2)
+      # user search
+      expect(mocked_ldap).to receive(:search).and_yield(lost_entry)
+
+      described_class.import(
+        config: config,
+        ldap:   mocked_ldap,
+      )
+
+      # activate skipping
+      config[:unassigned_users] = 'skip_sync'
+      config[:group_role_map]   = {
+        'dummy' => %w(1 2),
+      }
+
+      # group user role mapping
+      mocked_entry           = build(:ldap_entry)
+      mocked_entry['dn']     = 'dummy'
+      mocked_entry['member'] = ['dummy']
+      expect(mocked_ldap).to receive(:search).and_yield(mocked_entry)
+
+      # user counting
+      expect(mocked_ldap).to receive(:count).and_return(1)
+      # user search
+      expect(mocked_ldap).to receive(:search).and_yield(lost_entry)
+
+      expect do
+        described_class.import(
+          config: config,
+          ldap:   mocked_ldap,
+        )
+      end.to change {
+        User.find_by(email: 'example@example.com').active
+      }
+    end
+
+    context 'dry run' do
+
+      it "doesn't sync users" do
 
         config = {
           user_filter:     '(objectClass=user)',
@@ -90,6 +278,65 @@ RSpec.describe Import::Ldap::UserFactory do
           User.count
         }
       end
+
+      it "doesn't deactivates lost users" do
+
+        config = {
+          user_filter:     '(objectClass=user)',
+          group_filter:    '(objectClass=group)',
+          user_uid:        'uid',
+          user_attributes: {
+            'uid' => 'login',
+            'email' => 'email',
+          }
+        }
+
+        persistent_entry          = build(:ldap_entry)
+        persistent_entry['uid']   = ['exampleuid']
+        persistent_entry['email'] = ['example@example.com']
+
+        lost_entry          = build(:ldap_entry)
+        lost_entry['uid']   = ['exampleuid']
+        lost_entry['email'] = ['example@example.com']
+
+        mocked_ldap = double(
+          host:    'ldap.example.com',
+          port:    636,
+          ssl:     true,
+          base_dn: 'dc=example,dc=com'
+        )
+
+        # group user role mapping
+        expect(mocked_ldap).to receive(:search)
+        # user counting
+        expect(mocked_ldap).to receive(:count).and_return(2)
+        # user search
+        expect(mocked_ldap).to receive(:search).and_yield(persistent_entry).and_yield(lost_entry)
+
+        described_class.import(
+          config:  config,
+          ldap:    mocked_ldap,
+          dry_run: true
+        )
+
+        # group user role mapping
+        expect(mocked_ldap).to receive(:search)
+        # user counting
+        expect(mocked_ldap).to receive(:count).and_return(1)
+        # user search
+        expect(mocked_ldap).to receive(:search).and_yield(persistent_entry)
+
+        expect do
+          described_class.import(
+            config:  config,
+            ldap:    mocked_ldap,
+            dry_run: true
+          )
+        end.not_to change {
+          User.count
+        }
+      end
+    end
   end
 
   describe '.add_to_statistics' do
@@ -118,13 +365,15 @@ RSpec.describe Import::Ldap::UserFactory do
             created:     1,
             updated:     0,
             unchanged:   0,
-            failed:    0
+            failed:      0,
+            deactivated: 0,
           },
           2 => {
             created:     1,
             updated:     0,
             unchanged:   0,
-            failed:    0
+            failed:      0,
+            deactivated: 0,
           },
         },
         skipped:     0,
@@ -132,6 +381,72 @@ RSpec.describe Import::Ldap::UserFactory do
         updated:     0,
         unchanged:   0,
         failed:      0,
+        deactivated: 0,
+      }
+
+      expect(described_class.statistics).to include(expected)
+    end
+
+    it 'adds deactivated users' do
+      config = {
+        user_filter:     '(objectClass=user)',
+        group_filter:    '(objectClass=group)',
+        user_uid:        'uid',
+        user_attributes: {
+          'uid' => 'login',
+          'email' => 'email',
+        }
+      }
+
+      persistent_entry          = build(:ldap_entry)
+      persistent_entry['uid']   = ['exampleuid']
+      persistent_entry['email'] = ['example@example.com']
+
+      lost_entry          = build(:ldap_entry)
+      lost_entry['uid']   = ['exampleuid_lost']
+      lost_entry['email'] = ['lost@example.com']
+
+      mocked_ldap = double(
+        host:    'ldap.example.com',
+        port:    636,
+        ssl:     true,
+        base_dn: 'dc=example,dc=com'
+      )
+
+      # group user role mapping
+      expect(mocked_ldap).to receive(:search)
+      # user counting
+      allow(mocked_ldap).to receive(:count).and_return(2)
+      # user search
+      expect(mocked_ldap).to receive(:search).and_yield(persistent_entry).and_yield(lost_entry)
+
+      described_class.import(
+        config: config,
+        ldap:   mocked_ldap,
+      )
+
+      # simulate new import
+      described_class.reset_statistics
+
+      # group user role mapping
+      expect(mocked_ldap).to receive(:search)
+      # user counting
+      allow(mocked_ldap).to receive(:count).and_return(1)
+      # user search
+      expect(mocked_ldap).to receive(:search).and_yield(persistent_entry)
+
+      described_class.import(
+        config: config,
+        ldap:   mocked_ldap,
+      )
+
+      expected = {
+        skipped:     0,
+        created:     0,
+        updated:     0,
+        unchanged:   1,
+        failed:      0,
+        deactivated: 1,
       }
 
       expect(described_class.statistics).to include(expected)
@@ -155,6 +470,7 @@ RSpec.describe Import::Ldap::UserFactory do
         updated:     0,
         unchanged:   0,
         failed:      0,
+        deactivated: 0,
       }
 
       expect(described_class.statistics).to include(expected)
@@ -180,6 +496,7 @@ RSpec.describe Import::Ldap::UserFactory do
         updated:     0,
         unchanged:   0,
         failed:      0,
+        deactivated: 0,
       }
 
       expect(described_class.statistics).to include(expected)

spec/lib/import/statistical_factory_spec.rb

@@ -50,6 +50,7 @@ RSpec.describe Import::StatisticalFactory do
           unchanged:   0,
           skipped:     0,
           failed:      0,
+          deactivated: 0,
         }
         expect(Import::Test::GroupFactory.statistics).to eq(statistics)
       end
@@ -72,6 +73,7 @@ RSpec.describe Import::StatisticalFactory do
             unchanged:   0,
             skipped:     0,
             failed:      0,
+            deactivated: 0,
           }
           expect(Import::Test::GroupFactory.statistics).to eq(statistics)
         end
@@ -95,6 +97,7 @@ RSpec.describe Import::StatisticalFactory do
             unchanged:   0,
             skipped:     0,
             failed:      0,
+            deactivated: 0,
           }
           expect(Import::Test::GroupFactory.statistics).to eq(statistics)
         end
@@ -118,6 +121,7 @@ RSpec.describe Import::StatisticalFactory do
             unchanged:   0,
             skipped:     0,
             failed:      0,
+            deactivated: 0,
           }
           expect(Import::Test::GroupFactory.statistics).to eq(statistics)
         end
@@ -139,6 +143,7 @@ RSpec.describe Import::StatisticalFactory do
           unchanged:   1,
           skipped:     0,
           failed:      0,
+          deactivated: 0,
         }
         expect(Import::Test::GroupFactory.statistics).to eq(statistics)
       end
@@ -156,6 +161,7 @@ RSpec.describe Import::StatisticalFactory do
           unchanged:   0,
           skipped:     0,
           failed:      0,
+          deactivated: 0,
         }
         expect(Import::Test::GroupFactory.statistics).to eq(statistics)
       end
@@ -186,6 +192,7 @@ RSpec.describe Import::StatisticalFactory do
             unchanged:   0,
             skipped:     0,
             failed:      0,
+            deactivated: 0,
           }
           expect(Import::Test::GroupFactory.statistics).to eq(statistics)
         end
@@ -204,6 +211,7 @@ RSpec.describe Import::StatisticalFactory do
             unchanged:   0,
             skipped:     0,
             failed:      0,
+            deactivated: 0,
           }
           expect(Import::Test::GroupFactory.statistics).to eq(statistics)
         end
@@ -222,6 +230,7 @@ RSpec.describe Import::StatisticalFactory do
             unchanged:   0,
             skipped:     0,
             failed:      0,
+            deactivated: 0,
           }
           expect(Import::Test::GroupFactory.statistics).to eq(statistics)
         end
@@ -246,6 +255,7 @@ RSpec.describe Import::StatisticalFactory do
           unchanged:   1,
           skipped:     0,
           failed:      0,
+          deactivated: 0,
         }
         expect(Import::Test::GroupFactory.statistics).to eq(statistics)
       end