Fixed issue #1211 - LDAP users that are lost don't get reflected into Zammad.

This commit is contained in:
Thorsten Eckel 2017-06-27 11:30:27 +02:00
parent 98c43f9090
commit 4ee181b5d6
10 changed files with 528 additions and 112 deletions

View file

@ -124,7 +124,7 @@ class Form extends App.Controller
if _.isEmpty(job)
@lastImport.html('')
return
countDone = job.result.created + job.result.updated + job.result.unchanged + job.result.skipped + job.result.failed
countDone = job.result.created + job.result.updated + job.result.unchanged + job.result.skipped + job.result.failed + job.result.deactivated
if !job.result.roles
job.result.roles = {}
for role_id, statistic of job.result.role_ids
@ -545,6 +545,8 @@ class ConnectionWizard extends App.WizardModal
total += job.result.unchanged
if job.result.updated
total += job.result.updated
if job.result.deactivated
total += job.result.deactivated
@$('.js-progress progress').attr('value', total)
@$('.js-progress progress').attr('max', job.result.sum)
if job.finished_at
@ -566,7 +568,7 @@ class ConnectionWizard extends App.WizardModal
for role_id, statistic of job.result.role_ids
role = App.Role.find(role_id)
job.result.roles[role.displayName()] = statistic
countDone = job.result.created + job.result.updated + job.result.unchanged + job.result.skipped
countDone = job.result.created + job.result.updated + job.result.unchanged + job.result.skipped + job.result.deactivated
@showSlide('js-try')
el = $(App.view('integration/ldap_summary')(job: job, countDone: countDone))
@el.find('.js-summary').html(el)

View file

@ -33,13 +33,13 @@
<ul>
<li><%- @T('%s user to %s user', 'LDAP', 'Zammad') %> (<%= @countDone %>/<%= @job.result.sum %>):
<ul>
<li><%- @T('Users') %>: <%= @job.result.created %> <%- @T('created') %>, <%= @job.result.updated %> <%- @T('updated') %>, <%= @job.result.unchanged %> <%- @T('untouched') %>, <%= @job.result.skipped %> <%- @T('skipped') %>, <%= @job.result.failed %> <%- @T('failed') %>
<li><%- @T('Users') %>: <%= @job.result.created %> <%- @T('created') %>, <%= @job.result.updated %> <%- @T('updated') %>, <%= @job.result.unchanged %> <%- @T('untouched') %>, <%= @job.result.skipped %> <%- @T('skipped') %>, <%= @job.result.failed %> <%- @T('failed') %>, <%= @job.result.deactivated %> <%- @T('deactivated') %>
</ul>
<% if !_.isEmpty(@job.result.roles): %>
<li><%- @T('%s groups to %s roles assignments', 'LDAP', 'Zammad') %>:
<ul>
<% for role, result of @job.result.roles: %>
<li> <%- @T(role) %>: <%= result.created %> <%- @T('created') %>, <%= result.updated %> <%- @T('updated') %>, <%= result.unchanged %> <%- @T('untouched') %>, <%= result.failed %> <%- @T('failed') %>
<li> <%- @T(role) %>: <%= result.created %> <%- @T('created') %>, <%= result.updated %> <%- @T('updated') %>, <%= result.unchanged %> <%- @T('untouched') %>, <%= result.failed %> <%- @T('failed') %>, <%= result.deactivated %> <%- @T('deactivated') %>
<% end %>
</ul>
<% end %>

View file

@ -1,14 +1,14 @@
<ul>
<li><%- @T('%s user to %s user', 'LDAP', 'Zammad') %> (<%= @countDone %>):
<ul>
<li><%- @T('Users') %>: <%= @job.result.created %> <%- @T('created') %>, <%= @job.result.updated %> <%- @T('updated') %>, <%= @job.result.unchanged %> <%- @T('untouched') %>, <%= @job.result.skipped %> <%- @T('skipped') %>, <%= @job.result.failed %> <%- @T('failed') %>
<li><%- @T('Users') %>: <%= @job.result.created %> <%- @T('created') %>, <%= @job.result.updated %> <%- @T('updated') %>, <%= @job.result.unchanged %> <%- @T('untouched') %>, <%= @job.result.skipped %> <%- @T('skipped') %>, <%= @job.result.failed %> <%- @T('failed') %>, <%= @job.result.deactivated %> <%- @T('deactivated') %>
</ul>
</li>
<% if !_.isEmpty(@job.result.roles): %>
<li><%- @T('%s groups to %s roles assignments', 'LDAP', 'Zammad') %>:
<ul>
<% for role, result of @job.result.roles: %>
<li><%- @T(role) %>: <%= result.created %> <%- @T('created') %>, <%= result.updated %> <%- @T('updated') %>, <%= result.unchanged %> <%- @T('untouched') %>, <%= result.failed %> <%- @T('failed') %>
<li><%- @T(role) %>: <%= result.created %> <%- @T('created') %>, <%= result.updated %> <%- @T('updated') %>, <%= result.unchanged %> <%- @T('untouched') %>, <%= result.failed %> <%- @T('failed') %>, <%= result.deactivated %> <%- @T('deactivated') %>
<% end %>
</ul>
</li>

View file

@ -16,7 +16,7 @@ module Import
end
def source
import_class_namespace
self.class.source
end
def remote_id(resource, *_args)
@ -57,6 +57,14 @@ module Import
changes
end
def self.source
import_class_namespace
end
def self.import_class_namespace
@import_class_namespace ||= name.to_s.sub('Import::', '')
end
private
def initialize_associations_states
@ -214,11 +222,7 @@ module Import
end
def mapping_config(*_args)
import_class_namespace.gsub('::', '_').underscore + '_mapping'
end
def import_class_namespace
self.class.name.to_s.sub('Import::', '')
self.class.import_class_namespace.gsub('::', '_').underscore + '_mapping'
end
def handle_args(_resource, *args)

View file

@ -6,6 +6,35 @@ module Import
@remote_id
end
def self.lost_ids(found_remote_ids)
ExternalSync.joins('INNER JOIN users ON (users.id = external_syncs.o_id)')
.where(
source: source,
object: import_class.name,
users: {
active: true
}
)
.pluck(:source_id, :o_id)
.to_h
.except(*found_remote_ids)
.values
end
def self.deactivate_lost(lost_ids)
# we need to update in slices since some DBs
# have a limit for IN length
lost_ids.each_slice(5000) do |slice|
# we need to instanciate every entry and set
# the active state this way to send notifications
# to the client
::User.where(id: slice).each do |user|
user.update_attribute(:active, false)
end
end
end
private
def import(resource, *args)
@ -58,7 +87,7 @@ module Import
return true if resource[:login].blank?
# skip resource if only ignored attributes are set
ignored_attributes = %i(login dn created_by_id updated_by_id)
ignored_attributes = %i(login dn created_by_id updated_by_id active)
!resource.except(*ignored_attributes).values.any?(&:present?)
end
@ -181,6 +210,11 @@ module Import
mapped[attribute] = mapped[attribute].downcase
end
# we have to add the active state manually
# because otherwise disabled instances won't get
# re-activated if they should get synced again
mapped[:active] = true
mapped
end

View file

@ -33,10 +33,13 @@ module Import
relevant_attributes = config[:user_attributes].keys
relevant_attributes.push('dn')
@found_remote_ids = []
@ldap.search(config[:user_filter], attributes: relevant_attributes) do |entry|
backend_instance = create_instance(entry, config, user_roles, signup_role_ids, kargs)
post_import_hook(entry, backend_instance, config, user_roles, signup_role_ids, kargs)
track_found_remote_ids(backend_instance)
next if import_job.blank?
import_job_count += 1
next if import_job_count < 100
@ -47,6 +50,7 @@ module Import
import_job_count = 0
end
handle_lost
end
def self.pre_import_hook(_records, *_args)
@ -77,18 +81,25 @@ module Import
action = backend_instance.action
add_resource_role_ids_to_statistics(resource.role_ids, action)
action
end
def self.add_resource_role_ids_to_statistics(role_ids, action)
return if role_ids.blank?
known_actions = {
created: 0,
updated: 0,
unchanged: 0,
failed: 0,
created: 0,
updated: 0,
unchanged: 0,
failed: 0,
deactivated: 0,
}
if !@statistics[:role_ids]
@statistics[:role_ids] = {}
end
@statistics[:role_ids] ||= {}
resource.role_ids.each do |role_id|
role_ids.each do |role_id|
next if !known_actions.key?(action)
@ -99,8 +110,6 @@ module Import
@statistics[:role_ids][role_id][action] += 1
end
action
end
def self.user_roles(ldap:, config:)
@ -111,6 +120,37 @@ module Import
ldap_group = ::Ldap::Group.new(group_config, ldap: ldap)
ldap_group.user_roles(config[:group_role_map])
end
def self.track_found_remote_ids(backend_instance)
@deactivation_actions ||= %i(skipped failed)
return if @deactivation_actions.include?(backend_instance.action)
@found_remote_ids.push(backend_instance.remote_id(nil))
end
def self.handle_lost
backend_class = backend_class(nil)
lost_ids = backend_class.lost_ids(@found_remote_ids)
# track disabled count and substract it from
# skipped where they are logged till now
@statistics[:deactivated] = lost_ids.size
@statistics[:skipped] -= lost_ids.size
# loop over every lost user ID and add the
# deactivated count to the statistics
lost_ids.each do |user_id|
role_ids = ::User.joins(:roles)
.where(id: user_id)
.pluck(:'roles_users.role_id')
add_resource_role_ids_to_statistics(role_ids, :deactivated)
end
# deactivate entries only on live syncs
return if @dry_run
backend_class.deactivate_lost(lost_ids)
end
end
end
end

View file

@ -2,11 +2,19 @@ module Import
class ModelResource < Import::BaseResource
def import_class
model_name.constantize
self.class.import_class
end
def model_name
@model_name ||= self.class.name.split('::').last
self.class.model_name
end
def self.import_class
model_name.constantize
end
def self.model_name
@model_name ||= name.split('::').last
end
private

View file

@ -13,11 +13,12 @@ module Import
def reset_statistics
@statistics = {
skipped: 0,
created: 0,
updated: 0,
unchanged: 0,
failed: 0,
skipped: 0,
created: 0,
updated: 0,
unchanged: 0,
failed: 0,
deactivated: 0,
}
end

View file

@ -49,22 +49,25 @@ RSpec.describe Import::Ldap::UserFactory do
}.by(1)
end
it 'supports dry run' do
it 'deactivates lost users' do
config = {
user_filter: '(objectClass=user)',
group_filter: '(objectClass=group)',
user_uid: 'uid',
user_attributes: {
'uid' => 'login',
'uid' => 'login',
'email' => 'email',
}
}
mocked_entry = build(:ldap_entry)
persistent_entry = build(:ldap_entry)
persistent_entry['uid'] = ['exampleuid']
persistent_entry['email'] = ['example@example.com']
mocked_entry['uid'] = ['exampleuid']
mocked_entry['email'] = ['example@example.com']
lost_entry = build(:ldap_entry)
lost_entry['uid'] = ['exampleuid_lost']
lost_entry['email'] = ['lost@example.com']
mocked_ldap = double(
host: 'ldap.example.com',
@ -73,22 +76,266 @@ RSpec.describe Import::Ldap::UserFactory do
base_dn: 'dc=example,dc=com'
)
# group user role mapping
expect(mocked_ldap).to receive(:search)
# user counting
expect(mocked_ldap).to receive(:count).and_return(2)
# user search
expect(mocked_ldap).to receive(:search).and_yield(persistent_entry).and_yield(lost_entry)
described_class.import(
config: config,
ldap: mocked_ldap,
)
# group user role mapping
expect(mocked_ldap).to receive(:search)
# user counting
expect(mocked_ldap).to receive(:count).and_return(1)
# user search
expect(mocked_ldap).to receive(:search).and_yield(mocked_entry)
expect(mocked_ldap).to receive(:search).and_yield(persistent_entry)
expect do
described_class.import(
config: config,
ldap: mocked_ldap,
)
end.to change {
User.find_by(email: 'lost@example.com').active
}
end
it 're-activates previously lost users' do
config = {
user_filter: '(objectClass=user)',
group_filter: '(objectClass=group)',
user_uid: 'uid',
user_attributes: {
'uid' => 'login',
'email' => 'email',
}
}
persistent_entry = build(:ldap_entry)
persistent_entry['uid'] = ['exampleuid']
persistent_entry['email'] = ['example@example.com']
lost_entry = build(:ldap_entry)
lost_entry['uid'] = ['exampleuid_lost']
lost_entry['email'] = ['lost@example.com']
mocked_ldap = double(
host: 'ldap.example.com',
port: 636,
ssl: true,
base_dn: 'dc=example,dc=com'
)
# group user role mapping
expect(mocked_ldap).to receive(:search)
# user counting
expect(mocked_ldap).to receive(:count).and_return(2)
# user search
expect(mocked_ldap).to receive(:search).and_yield(persistent_entry).and_yield(lost_entry)
described_class.import(
config: config,
ldap: mocked_ldap,
)
# group user role mapping
expect(mocked_ldap).to receive(:search)
# user counting
expect(mocked_ldap).to receive(:count).and_return(1)
# user search
expect(mocked_ldap).to receive(:search).and_yield(persistent_entry)
described_class.import(
config: config,
ldap: mocked_ldap,
)
# group user role mapping
expect(mocked_ldap).to receive(:search)
# user counting
expect(mocked_ldap).to receive(:count).and_return(2)
# user search
expect(mocked_ldap).to receive(:search).and_yield(persistent_entry).and_yield(lost_entry)
expect do
described_class.import(
config: config,
ldap: mocked_ldap,
)
end.to change {
User.find_by(email: 'lost@example.com').active
}
end
it 'deactivates skipped users' do
config = {
user_filter: '(objectClass=user)',
group_filter: '(objectClass=group)',
user_uid: 'uid',
user_attributes: {
'uid' => 'login',
'email' => 'email',
},
}
lost_entry = build(:ldap_entry)
lost_entry['uid'] = ['exampleuid']
lost_entry['email'] = ['example@example.com']
mocked_ldap = double(
host: 'ldap.example.com',
port: 636,
ssl: true,
base_dn: 'dc=example,dc=com'
)
# group user role mapping
expect(mocked_ldap).to receive(:search)
# user counting
expect(mocked_ldap).to receive(:count).and_return(2)
# user search
expect(mocked_ldap).to receive(:search).and_yield(lost_entry)
described_class.import(
config: config,
ldap: mocked_ldap,
)
# activate skipping
config[:unassigned_users] = 'skip_sync'
config[:group_role_map] = {
'dummy' => %w(1 2),
}
# group user role mapping
mocked_entry = build(:ldap_entry)
mocked_entry['dn'] = 'dummy'
mocked_entry['member'] = ['dummy']
expect(mocked_ldap).to receive(:search).and_yield(mocked_entry)
# user counting
expect(mocked_ldap).to receive(:count).and_return(1)
# user search
expect(mocked_ldap).to receive(:search).and_yield(lost_entry)
expect do
described_class.import(
config: config,
ldap: mocked_ldap,
)
end.to change {
User.find_by(email: 'example@example.com').active
}
end
context 'dry run' do
it "doesn't sync users" do
config = {
user_filter: '(objectClass=user)',
group_filter: '(objectClass=group)',
user_uid: 'uid',
user_attributes: {
'uid' => 'login',
'email' => 'email',
}
}
mocked_entry = build(:ldap_entry)
mocked_entry['uid'] = ['exampleuid']
mocked_entry['email'] = ['example@example.com']
mocked_ldap = double(
host: 'ldap.example.com',
port: 636,
ssl: true,
base_dn: 'dc=example,dc=com'
)
# group user role mapping
expect(mocked_ldap).to receive(:search)
# user counting
expect(mocked_ldap).to receive(:count).and_return(1)
# user search
expect(mocked_ldap).to receive(:search).and_yield(mocked_entry)
expect do
described_class.import(
config: config,
ldap: mocked_ldap,
dry_run: true
)
end.not_to change {
User.count
}
end
it "doesn't deactivates lost users" do
config = {
user_filter: '(objectClass=user)',
group_filter: '(objectClass=group)',
user_uid: 'uid',
user_attributes: {
'uid' => 'login',
'email' => 'email',
}
}
persistent_entry = build(:ldap_entry)
persistent_entry['uid'] = ['exampleuid']
persistent_entry['email'] = ['example@example.com']
lost_entry = build(:ldap_entry)
lost_entry['uid'] = ['exampleuid']
lost_entry['email'] = ['example@example.com']
mocked_ldap = double(
host: 'ldap.example.com',
port: 636,
ssl: true,
base_dn: 'dc=example,dc=com'
)
# group user role mapping
expect(mocked_ldap).to receive(:search)
# user counting
expect(mocked_ldap).to receive(:count).and_return(2)
# user search
expect(mocked_ldap).to receive(:search).and_yield(persistent_entry).and_yield(lost_entry)
described_class.import(
config: config,
ldap: mocked_ldap,
dry_run: true
)
end.not_to change {
User.count
}
# group user role mapping
expect(mocked_ldap).to receive(:search)
# user counting
expect(mocked_ldap).to receive(:count).and_return(1)
# user search
expect(mocked_ldap).to receive(:search).and_yield(persistent_entry)
expect do
described_class.import(
config: config,
ldap: mocked_ldap,
dry_run: true
)
end.not_to change {
User.count
}
end
end
end
@ -115,23 +362,91 @@ RSpec.describe Import::Ldap::UserFactory do
expected = {
role_ids: {
1 => {
created: 1,
updated: 0,
unchanged: 0,
failed: 0
created: 1,
updated: 0,
unchanged: 0,
failed: 0,
deactivated: 0,
},
2 => {
created: 1,
updated: 0,
unchanged: 0,
failed: 0
created: 1,
updated: 0,
unchanged: 0,
failed: 0,
deactivated: 0,
},
},
skipped: 0,
created: 1,
updated: 0,
unchanged: 0,
failed: 0,
skipped: 0,
created: 1,
updated: 0,
unchanged: 0,
failed: 0,
deactivated: 0,
}
expect(described_class.statistics).to include(expected)
end
it 'adds deactivated users' do
config = {
user_filter: '(objectClass=user)',
group_filter: '(objectClass=group)',
user_uid: 'uid',
user_attributes: {
'uid' => 'login',
'email' => 'email',
}
}
persistent_entry = build(:ldap_entry)
persistent_entry['uid'] = ['exampleuid']
persistent_entry['email'] = ['example@example.com']
lost_entry = build(:ldap_entry)
lost_entry['uid'] = ['exampleuid_lost']
lost_entry['email'] = ['lost@example.com']
mocked_ldap = double(
host: 'ldap.example.com',
port: 636,
ssl: true,
base_dn: 'dc=example,dc=com'
)
# group user role mapping
expect(mocked_ldap).to receive(:search)
# user counting
allow(mocked_ldap).to receive(:count).and_return(2)
# user search
expect(mocked_ldap).to receive(:search).and_yield(persistent_entry).and_yield(lost_entry)
described_class.import(
config: config,
ldap: mocked_ldap,
)
# simulate new import
described_class.reset_statistics
# group user role mapping
expect(mocked_ldap).to receive(:search)
# user counting
allow(mocked_ldap).to receive(:count).and_return(1)
# user search
expect(mocked_ldap).to receive(:search).and_yield(persistent_entry)
described_class.import(
config: config,
ldap: mocked_ldap,
)
expected = {
skipped: 0,
created: 0,
updated: 0,
unchanged: 1,
failed: 0,
deactivated: 1,
}
expect(described_class.statistics).to include(expected)
@ -150,11 +465,12 @@ RSpec.describe Import::Ldap::UserFactory do
described_class.add_to_statistics(mocked_backend_instance)
expected = {
skipped: 1,
created: 0,
updated: 0,
unchanged: 0,
failed: 0,
skipped: 1,
created: 0,
updated: 0,
unchanged: 0,
failed: 0,
deactivated: 0,
}
expect(described_class.statistics).to include(expected)
@ -175,11 +491,12 @@ RSpec.describe Import::Ldap::UserFactory do
described_class.add_to_statistics(mocked_backend_instance)
expected = {
skipped: 1,
created: 0,
updated: 0,
unchanged: 0,
failed: 0,
skipped: 1,
created: 0,
updated: 0,
unchanged: 0,
failed: 0,
deactivated: 0,
}
expect(described_class.statistics).to include(expected)

View file

@ -45,11 +45,12 @@ RSpec.describe Import::StatisticalFactory do
Import::Test::GroupFactory.import([attributes])
statistics = {
created: 1,
updated: 0,
unchanged: 0,
skipped: 0,
failed: 0,
created: 1,
updated: 0,
unchanged: 0,
skipped: 0,
failed: 0,
deactivated: 0,
}
expect(Import::Test::GroupFactory.statistics).to eq(statistics)
end
@ -67,11 +68,12 @@ RSpec.describe Import::StatisticalFactory do
Import::Test::GroupFactory.import([attributes])
statistics = {
created: 0,
updated: 1,
unchanged: 0,
skipped: 0,
failed: 0,
created: 0,
updated: 1,
unchanged: 0,
skipped: 0,
failed: 0,
deactivated: 0,
}
expect(Import::Test::GroupFactory.statistics).to eq(statistics)
end
@ -90,11 +92,12 @@ RSpec.describe Import::StatisticalFactory do
Import::Test::GroupFactory.import([attributes])
statistics = {
created: 0,
updated: 1,
unchanged: 0,
skipped: 0,
failed: 0,
created: 0,
updated: 1,
unchanged: 0,
skipped: 0,
failed: 0,
deactivated: 0,
}
expect(Import::Test::GroupFactory.statistics).to eq(statistics)
end
@ -113,11 +116,12 @@ RSpec.describe Import::StatisticalFactory do
Import::Test::GroupFactory.import([attributes])
statistics = {
created: 0,
updated: 1,
unchanged: 0,
skipped: 0,
failed: 0,
created: 0,
updated: 1,
unchanged: 0,
skipped: 0,
failed: 0,
deactivated: 0,
}
expect(Import::Test::GroupFactory.statistics).to eq(statistics)
end
@ -134,11 +138,12 @@ RSpec.describe Import::StatisticalFactory do
Import::Test::GroupFactory.import([attributes])
statistics = {
created: 0,
updated: 0,
unchanged: 1,
skipped: 0,
failed: 0,
created: 0,
updated: 0,
unchanged: 1,
skipped: 0,
failed: 0,
deactivated: 0,
}
expect(Import::Test::GroupFactory.statistics).to eq(statistics)
end
@ -151,11 +156,12 @@ RSpec.describe Import::StatisticalFactory do
Import::Test::GroupFactory.import([attributes], dry_run: true)
statistics = {
created: 1,
updated: 0,
unchanged: 0,
skipped: 0,
failed: 0,
created: 1,
updated: 0,
unchanged: 0,
skipped: 0,
failed: 0,
deactivated: 0,
}
expect(Import::Test::GroupFactory.statistics).to eq(statistics)
end
@ -181,11 +187,12 @@ RSpec.describe Import::StatisticalFactory do
Import::Test::GroupFactory.import([update_attributes], dry_run: true)
statistics = {
created: 0,
updated: 1,
unchanged: 0,
skipped: 0,
failed: 0,
created: 0,
updated: 1,
unchanged: 0,
skipped: 0,
failed: 0,
deactivated: 0,
}
expect(Import::Test::GroupFactory.statistics).to eq(statistics)
end
@ -199,11 +206,12 @@ RSpec.describe Import::StatisticalFactory do
Import::Test::GroupFactory.import([update_attributes], dry_run: true)
statistics = {
created: 0,
updated: 1,
unchanged: 0,
skipped: 0,
failed: 0,
created: 0,
updated: 1,
unchanged: 0,
skipped: 0,
failed: 0,
deactivated: 0,
}
expect(Import::Test::GroupFactory.statistics).to eq(statistics)
end
@ -217,11 +225,12 @@ RSpec.describe Import::StatisticalFactory do
Import::Test::GroupFactory.import([update_attributes], dry_run: true)
statistics = {
created: 0,
updated: 1,
unchanged: 0,
skipped: 0,
failed: 0,
created: 0,
updated: 1,
unchanged: 0,
skipped: 0,
failed: 0,
deactivated: 0,
}
expect(Import::Test::GroupFactory.statistics).to eq(statistics)
end
@ -241,11 +250,12 @@ RSpec.describe Import::StatisticalFactory do
Import::Test::GroupFactory.import([local_group.attributes], dry_run: true)
statistics = {
created: 0,
updated: 0,
unchanged: 1,
skipped: 0,
failed: 0,
created: 0,
updated: 0,
unchanged: 1,
skipped: 0,
failed: 0,
deactivated: 0,
}
expect(Import::Test::GroupFactory.statistics).to eq(statistics)
end