Added LDAP sync Sequencer Sequences and Units.

lib/sequencer/sequence/import/ldap/user.rb

@@ -0,0 +1,42 @@
+class Sequencer
+  class Sequence
+    module Import
+      module Ldap
+        class User < Sequencer::Sequence::Base
+
+          def self.expecting
+            [:instance]
+          end
+
+          def self.sequence
+            [
+              'Import::Ldap::User::NormalizeEntry',
+              'Import::Ldap::User::RemoteId',
+              'Import::Ldap::User::Mapping',
+              'Import::Ldap::User::Skip::MissingMandatory',
+              'Import::Ldap::User::Skip::Blank',
+              'Import::Common::Model::Lookup::ExternalSync',
+              'Import::Common::User::Attributes::Downcase',
+              'Import::Common::User::Email::CheckValidity',
+              'Import::Ldap::User::Lookup::Attributes',
+              'Import::Ldap::User::Attributes::RoleIds::Dn',
+              'Import::Ldap::User::Attributes::RoleIds::Unassigned',
+              'Import::Common::Model::Associations::Extract',
+              'Import::Ldap::User::Attributes::Static',
+              'Import::Common::Model::Attributes::AddByIds',
+              'Import::Common::Model::Update',
+              'Import::Common::Model::Create',
+              'Import::Common::Model::Associations::Assign',
+              'Import::Ldap::User::Model::Save',
+              'Import::Common::Model::ExternalSync::Integrity',
+              'Import::Ldap::User::HttpLog',
+              'Import::Ldap::User::Statistics::Diff',
+              'Import::Common::ImportJob::Statistics::Update',
+              'Import::Common::ImportJob::Statistics::Store',
+            ]
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/sequence/import/ldap/users.rb

@@ -0,0 +1,27 @@
+class Sequencer
+  class Sequence
+    module Import
+      module Ldap
+        class Users < Sequencer::Sequence::Base
+
+          def self.sequence
+            [
+              'Import::Ldap::Users::StaticAttributes',
+              'Import::Ldap::Users::DryRun::Flag',
+              'Import::Ldap::Users::DryRun::Payload',
+              'Ldap::Connection',
+              'Import::Ldap::Users::UserRoles',
+              'Import::Ldap::Users::Sum',
+              'Import::Common::ImportJob::Statistics::Update',
+              'Import::Common::ImportJob::Statistics::Store',
+              'Import::Ldap::Users::SubSequence',
+              'Import::Ldap::Users::Lost::Ids',
+              'Import::Ldap::Users::Lost::StatisticsDiff',
+              'Import::Ldap::Users::Lost::Deactivate',
+            ]
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/user/attributes/role_ids/dn.rb

@@ -0,0 +1,46 @@
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module User
+          module Attributes
+            module RoleIds
+              class Dn < Sequencer::Unit::Base
+                include ::Sequencer::Unit::Import::Common::Mapping::Mixin::ProvideMapped
+                prepend ::Sequencer::Unit::Import::Common::Model::Mixin::Skip::InstanceAction
+
+                skip_any_instance_action
+
+                uses :resource, :remote_id, :dn_roles
+
+                def process
+                  dn = resource[:dn]
+                  raise "Missing 'dn' attribute for remote id '#{remote_id}'" if dn.blank?
+
+                  # use signup/Zammad default roles
+                  # if no mapping was provided
+                  return if dn_roles.blank?
+
+                  # check if roles are mapped for the found dn
+                  role_ids = dn_roles[ dn.downcase ]
+
+                  # use signup/Zammad default roles
+                  # if no mapping entry was found
+                  return if role_ids.blank?
+
+                  # LDAP is the leading source if
+                  # a mapping entry is present
+                  provide_mapped do
+                    {
+                      role_ids: role_ids
+                    }
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/user/attributes/role_ids/unassigned.rb

@@ -0,0 +1,47 @@
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module User
+          module Attributes
+            module RoleIds
+              class Unassigned < Sequencer::Unit::Base
+                prepend ::Sequencer::Unit::Import::Common::Model::Mixin::Skip::InstanceAction
+
+                skip_any_instance_action
+
+                uses :resource, :dn_roles, :ldap_config, :mapped
+                provides :instance_action
+
+                def process
+                  # use signup/Zammad default roles
+                  # if no mapping was provided
+                  return if dn_roles.blank?
+
+                  # return if a mapping entry was found
+                  return if mapped[:role_ids].present?
+
+                  # use signup/Zammad default roles
+                  # if unassigned users should not get skipped
+                  return if ldap_config[:unassigned_users] != 'skip_sync'
+
+                  instance = state.optional(:instance)
+
+                  if instance.present?
+                    # deactivate instance if role assignment is lost
+                    instance.update!(active: false)
+                    state.provide(:instance_action, :deactivated)
+                  else
+                    # skip instance creation if no existing
+                    # instance was found yet
+                    state.provide(:instance_action, :skipped)
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/user/attributes/static.rb

@@ -0,0 +1,29 @@
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module User
+          module Attributes
+            class Static < Sequencer::Unit::Base
+              include ::Sequencer::Unit::Import::Common::Mapping::Mixin::ProvideMapped
+              prepend ::Sequencer::Unit::Import::Common::Model::Mixin::Skip::InstanceAction
+
+              skip_any_instance_action
+
+              def process
+                provide_mapped do
+                  {
+                    # we have to add the active state manually
+                    # because otherwise disabled instances won't get
+                    # re-activated if they should get synced again
+                    active: true,
+                  }
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/user/http_log.rb

@@ -0,0 +1,17 @@
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module User
+          class HttpLog < Import::Common::Model::HttpLog
+            private
+
+            def facility
+              'ldap'
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/user/lookup/attributes.rb

@@ -0,0 +1,19 @@
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module User
+          module Lookup
+            class Attributes < Sequencer::Unit::Import::Common::Model::Lookup::Attributes
+              private
+
+              def attributes
+                %i[login email]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/user/mapping.rb

@@ -0,0 +1,25 @@
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module User
+          class Mapping < Sequencer::Unit::Import::Common::Mapping::FlatKeys
+            uses :ldap_config
+
+            private
+
+            def mapping
+              ldap_config[:user_attributes].dup.tap do |config|
+                # fallback to uid as login
+                # if no login is given via mapping
+                if !config.values.include?('login')
+                  config[ ldap_config[:user_uid] ] = 'login'
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/user/model/save.rb

@@ -0,0 +1,20 @@
+require 'sequencer/unit/import/common/model/mixin/without_callback'
+
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module User
+          module Model
+            class Save < Import::Common::Model::Save
+              prepend ::Sequencer::Unit::Import::Common::Model::Mixin::WithoutCallback
+
+              without_callback :create, :after, :avatar_for_email_check
+              without_callback :update, :after, :avatar_for_email_check
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/user/normalize_entry.rb

@@ -0,0 +1,24 @@
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module User
+          class NormalizeEntry < Sequencer::Unit::Base
+            uses :resource
+            provides :resource
+
+            def process
+
+              state.provide(:resource) do
+                empty = ActiveSupport::HashWithIndifferentAccess.new
+                resource.each_with_object(empty) do |(key, values), normalized|
+                  normalized[key] = values.first
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/user/remote_id.rb

@@ -0,0 +1,19 @@
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module User
+          class RemoteId < Sequencer::Unit::Import::Common::Model::Attributes::RemoteId
+            uses :ldap_config
+
+            private
+
+            def attribute
+              ldap_config[:user_uid].to_sym
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/user/skip/blank.rb

@@ -0,0 +1,19 @@
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module User
+          module Skip
+            class Blank < Sequencer::Unit::Import::Common::Model::Skip::Blank::Mapped
+              private
+
+              def ignore
+                %i[login]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/user/skip/missing_mandatory.rb

@@ -0,0 +1,19 @@
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module User
+          module Skip
+            class MissingMandatory < Sequencer::Unit::Import::Common::Model::Skip::MissingMandatory::Mapped
+              private
+
+              def mandatory
+                [:login]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/user/statistics/diff.rb

@@ -0,0 +1,46 @@
+require 'sequencer/unit/import/common/model/statistics/mixin/instance_action_diff'
+
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module User
+          module Statistics
+            class Diff < Sequencer::Unit::Base
+              include ::Sequencer::Unit::Import::Common::Model::Statistics::Mixin::InstanceActionDiff
+
+              uses :instance, :associations, :signup_role_ids
+
+              def process
+                state.provide(:statistics_diff) do
+                  # remove :sum since it's already set via
+                  # the outer count Unit
+                  statistics = diff.except(:sum)
+
+                  add_role_ids(statistics)
+                end
+              end
+
+              private
+
+              def add_role_ids(statistics)
+                return statistics if instance.blank?
+
+                # add the parent role_ids hash
+                # so we can fill it
+                statistics[:role_ids] = {}
+
+                associations[:role_ids] ||= signup_role_ids
+
+                # add the diff for each role_id the user is assigned to
+                associations[:role_ids].each_with_object(statistics) do |role_id, result|
+                  result[:role_ids][role_id] = diff
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/users/dry_run/flag.rb

@@ -0,0 +1,20 @@
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module Users
+          class DryRun
+            class Flag < Sequencer::Unit::Base
+              uses :import_job
+              provides :dry_run
+
+              def process
+                state.provide(:dry_run, import_job.dry_run)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/users/dry_run/payload.rb

@@ -0,0 +1,15 @@
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module Users
+          class DryRun
+            class Payload < Sequencer::Unit::Import::Common::ImportJob::Payload::ToAttribute
+              provides :ldap_config
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/users/lost/deactivate.rb

@@ -0,0 +1,31 @@
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module Users
+          module Lost
+            class Deactivate < Sequencer::Unit::Base
+              uses :dry_run, :lost_ids
+
+              def process
+                return if dry_run
+
+                # we need to update in slices since some DBs
+                # have a limit for IN length
+                lost_ids.each_slice(5000) do |slice|
+
+                  # we need to instanciate every entry and set
+                  # the active state this way to send notifications
+                  # to the client
+                  ::User.where(id: slice).each do |user|
+                    user.update!(active: false)
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/users/lost/ids.rb

@@ -0,0 +1,32 @@
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module Users
+          module Lost
+            class Ids < Sequencer::Unit::Base
+              uses :found_ids, :external_sync_source, :model_class
+              provides :lost_ids
+
+              def process
+                state.provide(:lost_ids, active_ids - found_ids)
+              end
+
+              def active_ids
+                ExternalSync.joins('INNER JOIN users ON (users.id = external_syncs.o_id)')
+                            .where(
+                              source: external_sync_source,
+                              object: model_class.name,
+                              users:  {
+                                active: true
+                              }
+                            )
+                            .pluck(:o_id)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/users/lost/statistics_diff.rb

@@ -0,0 +1,44 @@
+require 'sequencer/unit/import/common/model/statistics/mixin/empty_diff'
+
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module Users
+          module Lost
+            class StatisticsDiff < Sequencer::Unit::Base
+              include ::Sequencer::Unit::Import::Common::Model::Statistics::Mixin::EmptyDiff
+
+              uses :lost_ids
+
+              def process
+                # deactivated count is tracked as a separate number
+                # since they don't have to be in the sum (e.g. deleted in LDAP)
+                state.provide(:statistics_diff) do
+                  diff.merge(
+                    role_ids:    role_ids,
+                    deactivated: lost_ids.size
+                  )
+                end
+              end
+
+              def role_ids
+                lost_ids.each_with_object({}) do |user_id, result|
+
+                  role_ids = ::User.joins(:roles)
+                                   .where(id: user_id)
+                                   .pluck(:'roles_users.role_id')
+
+                  role_ids.each do |role_id|
+                    result[role_id]               ||= diff
+                    result[role_id][:deactivated]  += 1
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/users/static_attributes.rb

@@ -0,0 +1,18 @@
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module Users
+          class StaticAttributes < Sequencer::Unit::Base
+            provides :model_class, :external_sync_source
+
+            def process
+              state.provide(:model_class, ::User)
+              state.provide(:external_sync_source, 'Ldap::User')
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/users/sub_sequence.rb

@@ -0,0 +1,58 @@
+require 'sequencer/unit/import/common/sub_sequence/mixin/import_job'
+
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module Users
+          class SubSequence < Sequencer::Unit::Base
+            include ::Sequencer::Unit::Import::Common::SubSequence::Mixin::ImportJob
+
+            uses :ldap_config, :ldap_connection, :dn_roles, :model_class, :external_sync_source
+            provides :found_ids
+
+            def process
+              found_ids = []
+              ldap_connection.search(ldap_config[:user_filter], attributes: relevant_attributes) do |entry|
+
+                result = sequence_resource(entry)
+
+                next if result[:instance].blank?
+                found_ids.push(result[:instance].id)
+              end
+
+              state.provide(:found_ids, found_ids)
+            end
+
+            private
+
+            def default_params
+              super.merge(
+                dn_roles:             dn_roles,
+                ldap_config:          ldap_config,
+                model_class:          model_class,
+                external_sync_source: external_sync_source,
+                signup_role_ids:      signup_role_ids
+              )
+            end
+
+            def signup_role_ids
+              @signup_role_ids ||= Role.signup_role_ids.sort
+            end
+
+            def sequence
+              'Import::Ldap::User'
+            end
+
+            def relevant_attributes
+              # limit the fetched attributes for an entry to only
+              # those which are needed to improve the performance
+              attributes = ldap_config[:user_attributes].keys
+              attributes.push('dn')
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/users/sum.rb

@@ -0,0 +1,45 @@
+require 'sequencer/unit/import/common/model/statistics/mixin/empty_diff'
+
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module Users
+          class Sum < Sequencer::Unit::Base
+            include ::Sequencer::Unit::Import::Common::Model::Statistics::Mixin::EmptyDiff
+
+            uses :ldap_config, :ldap_connection, :dry_run
+
+            def process
+              state.provide(:statistics_diff) do
+                diff.merge(
+                  sum: sum
+                )
+              end
+            end
+
+            private
+
+            def sum
+              if !dry_run
+                result = Cache.get(cache_key)
+              end
+
+              result ||= ldap_connection.count(ldap_config[:user_filter])
+
+              if !dry_run
+                Cache.write(cache_key, result, { expires_in: 1.hour })
+              end
+
+              result
+            end
+
+            def cache_key
+              @cache_key ||= "#{ldap_connection.host}::#{ldap_connection.port}::#{ldap_connection.ssl}::#{ldap_connection.base_dn}::#{ldap_config[:user_filter]}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/import/ldap/users/user_roles.rb

@@ -0,0 +1,31 @@
+require 'ldap'
+require 'ldap/group'
+
+class Sequencer
+  class Unit
+    module Import
+      module Ldap
+        module Users
+          class UserRoles < Sequencer::Unit::Base
+            uses :ldap_config, :ldap_connection
+            provides :dn_roles
+
+            def process
+
+              state.provide(:dn_roles) do
+
+                group_config = {
+                  filter: ldap_config[:group_filter]
+                }
+
+                ldap_group = ::Ldap::Group.new(group_config, ldap: ldap_connection)
+
+                ldap_group.user_roles(ldap_config[:group_role_map])
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/sequencer/unit/ldap/connection.rb

@@ -0,0 +1,24 @@
+require 'ldap'
+require 'import/ldap'
+
+class Sequencer
+  class Unit
+    module Ldap
+      class Connection < Sequencer::Unit::Base
+        uses :ldap_config
+        provides :ldap_connection
+
+        def process
+          return if state.provided?(:ldap_connection)
+
+          state.provide(:ldap_connection) do
+            config   = ldap_config
+            config ||= ::Import::Ldap.config
+
+            ::Ldap.new(config)
+          end
+        end
+      end
+    end
+  end
+end