[TASK] Introduces container mapping for the ldap loader

You should take a look at the documentation, because it is
too much for the commit message...

hshetl/loaders.py

@@ -316,20 +316,56 @@ class SqlAlchemyLoader(AbstractLoader):
 class LdapLoader(AbstractLoader):
     '''A loader for LDAP data sources using the ldap and ldap.modlist modules.
 
-    .. function:: __init__(rdn, base, objectClass[, **kwargs])
+    The LDAP loader supports container mapping in the following form:
+
+    - condition_handling is a Flag which can either be LOG or DEFAULT
+
+      - on DEFAULT there will be a default container mapping created from the rdn and base give in init
+      - on LOG there will be NO default container mapping created which can result in records that could be loaded, because no container mapping could be applied(!), but you will get warnings in the log
+    - move is a list of actions that should move rather than do their actual task
+
+      - if "update" is in the list, the update action will move an entry AFTER the actual update of the entry
+      - if "delete" is in the list, the delete action will move an entry INSTED OF deleting it
+    - You can define as many container mapping as you want
+    - container mappings consist of operation, condition, target and an optional rdn
+
+      - the container mapping "operation" is a list of operations for which this containerm mapping should be applied
+      - the container mapping "condition" is python code, that will be thrown into eval (you can ONLY use UNMAPPED property names in the condition!)
+      - the container mapping "target" is the base, where the record with the matching condition should be written or moved to
+      - the container mapping "rdn" is the rdn of the record, that will be the first part of the dn
+      - examples:
+
+        - condition:
+
+          - data['fooUnmapped'] == 'bar'
+          - True
+          - 'bar' in data['fooUnmapped']
+        - target + rdn:
+
+          - target: ou=foo,dc=de    rdn: sn    dn: sn=valueOfsnInRecord,ou=foo,dc=de
+
+    .. function:: __init__(rdn, objectClass[, base = None[, container_mapping = [][, condition_handling = 'LOG'[, move = [][, **kwargs]]]]])
 
         Constructor
 
         :param rdn: The attribute used to create a DN for new records.
-        :param base: The container in which new records will be created created.
         :param objectClass: The objectClass for new records - use this instead of providing it inside your entity.
+        :param base: The container in which new records will be created created.
+        :param container_mapping: The container mapping to be applied (priorised!!!)
+        :param condition_handling: Flag to determine, whether there should be a default mapping or not.
+        :param move: Determines, whether a record should be moved on update or delete, if a container mapping matches and WOULD move the record
         :param **kwargs: Accepts parameters from :class:`.AbstractExtractor`.
         :type rdn: string
-        :type base: string
         :type objectClass: string
+        :type base: string
+        :type container_mapping: list
+        :type condition_handling: string
+        :type move: list
         :type connector: :class:`hshetl.connectors.LdapConnector`
 
-    YAML definition sample:
+    YAML definition samples:
+
+    Deprecated:
 
     .. code-block:: yaml
 
@@ -339,12 +375,35 @@ class LdapLoader(AbstractLoader):
        base: ou=rz,o=fh-hannover
        objectClass: person
 
+    New:
+
+    .. code-block:: yaml
+
+        !ldaploader
+        connector: edirectory
+        rdn: cn
+        base: ou=rz,o=fh-hannover    (optional if you have a condition mapping with the condition default)
+        objectClass: person
+        condition_handling: LOG      (LOG or DEFAULT to create a default container mapping from the base)
+        move: ['update', 'delete']
+        container_mapping:
+          -
+            operation: ['insert', 'update', 'delete']
+            condition: data['l'] == 'Foo'
+            target: ou=foo,o=fh-hannover
+            rdn: sn                  (optional)
+          -
+            operation: ['insert']
+            condition: True
+            target: ou=new,o=fh-hannover
+
     '''
 
     yaml_tag = u'!ldaploader'
     '''Use this tag inside your YAML configuration, to define this loader.'''
 
-    def __init__(self, rdn, base, objectClass, **kwargs):
+    #TODO: remove base once conversion to transformers is complete
+    def __init__(self, rdn, objectClass, base = None, container_mapping = [], condition_handling = 'LOG', move = [], **kwargs):
         super(LdapLoader, self).__init__(**kwargs)
         self.rdn = rdn
         '''The attribute that is used to create new DNs.'''
@@ -352,6 +411,14 @@ class LdapLoader(AbstractLoader):
         '''The container new records will be inserted in.'''
         self.objectClass = objectClass
         '''The objectClass of new inserted records.'''
+        self.container_mapping = container_mapping
+        if None in (self.rdn, self.objectClass) or (self.base == None and self.container_mapping[0]['target'] == None):
+            raise ConfigurationException('base, rdn or objectClass not found - needs to be configured in the connector or loader!')
+        if condition_handling == 'DEFAULT' and {'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': self.rdn, 'target': self.base} not in container_mapping:
+            self.container_mapping.append({'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': self.rdn, 'target': self.base})
+        '''The container mappings for this loader'''
+        self.move = move
+        '''Move records or not'''
 
     def can_execute(self, connector):
         '''Defines which connector can be handled by this extractor.'''
@@ -360,8 +427,6 @@ class LdapLoader(AbstractLoader):
     def _execute(self, result):
         '''Executes the loading of data. Distinguishes between update, insert and delete.'''
         logging.debug('Loads data: ' + self.__class__.__name__)
-        if None in (self.rdn, self.base, self.objectClass):
-            raise ConfigurationException('base, rdn or objectClass not found - needs to be configured in the connector or loader!')
         with self.connector as connection:
             for action in self.operations:
                 getattr(self, '_' + action)(connection, getattr(result, action))
@@ -378,7 +443,9 @@ class LdapLoader(AbstractLoader):
             old_entity_dict = self._prepare_data(record.to_dict(container = record.last_container.target_container))
             dict_data = record.to_dict()
             entity_dict = self._prepare_data(dict_data)
-            dn = self._get_dn(record.get_container_identifier(), dict_data)
+            dn = self._get_dn(dict_data, 'update')
+            if dn == None: continue
+            old_dn = record.get_container_identifier()['dn'] if record.get_container_identifier() is not None else self.rdn + '=' + str(data[self.rdn]) + ',' + self.base
             attributes = modlist.modifyModlist(old_entity_dict, entity_dict)
             try:
                 if self.dry:
@@ -386,6 +453,12 @@ class LdapLoader(AbstractLoader):
                 else:
                     connection.modify_s(dn, attributes)
                     logging.info('Modified dn {} with {} '.format(dn, self._generate_modify_logging(attributes, old_entity_dict, entity_dict)))
+                if dn != old_dn:
+                    if 'update' in self.move:
+                        logging.info('Container %s will be moved to %s after update' % (old_dn, dn))
+                        self._move(connection, old_dn, dn)
+                    else:
+                        logging.info('Move after update prohibited by config (use move: [\'update\'])')
             except Exception, e:
                 logging.warn('Update failed for dn \'' + dn + '\': ' + str(e))
 
@@ -393,7 +466,8 @@ class LdapLoader(AbstractLoader):
         '''Inserts every dataset one after another.'''
         for record in data:
             entity_dict = self._prepare_data(record.to_dict())
-            dn = self._get_dn(record.get_container_identifier(), record.to_dict())
+            dn = self._get_dn(record.to_dict(), 'insert')
+            if dn == None: continue
             attributes = modlist.addModlist(entity_dict)
             try:
                 if self.dry:
@@ -407,7 +481,15 @@ class LdapLoader(AbstractLoader):
     def _delete(self, connection, data):
         '''Deletes every entry one after another.'''
         for record in data:
-            dn = self._get_dn(record.get_container_identifier(), record.to_dict())
+            dn = self._get_dn(record.to_dict(record.last_container.target_container), 'delete')
+            if dn == None: continue
+            old_dn = record.get_container_identifier()['dn'] if record.get_container_identifier() is not None else self.rdn + '=' + str(data[self.rdn]) + ',' + self.base
+            if dn != old_dn:
+                if 'delete' in self.move:
+                    logging.info('Container %s will be moved to %s instead of being deleted' % (old_dn, dn))
+                    self._move(connection, old_dn, dn)
+                else:
+                    logging.info('Move after delete prohibited by config (use move: [\'delete\'])')
             try:
                 if self.dry:
                     logging.info('Would delete dn {}'.format(dn))
@@ -417,13 +499,38 @@ class LdapLoader(AbstractLoader):
             except Exception, e:
                 logging.warn('Delete failed for dn \'' + dn + '\': ' + str(e))
 
-    def _get_dn(self, identifier, data):
-        if isinstance(identifier, dict) and 'dn' in identifier and len(identifier) == 1:
-            return identifier['dn']
-        elif identifier == None:
-            return self.rdn + '=' + str(data[self.rdn]) + ',' + self.base
+    def _move(self, connection, old_dn, new_dn):
+        '''Moves a single record'''
+        try:
+            if self.dry:
+                logging.info('Would move dn %s to %s' % (old_dn, new_dn))
+            else:
+                rdn = new_dn.split(',')[0]
+                newsuperior = new_dn[len(rdn)+1:]
+                connection.rename_s(old_dn, rdn, newsuperior)
+                logging.info('Moved dn %s to %s' % (old_dn, new_dn))
+        except Exception, e:
+            logging.warn('Move failed for old dn %s to new dn %s' % (old_dn, new_dn))
+
+    def _get_dn(self, data, operation):
+        '''Resolves the dn for the given record'''
+        possible_dns = []
+        for cmapping in self.container_mapping:
+            if not cmapping.has_key('rdn'):
+                cmapping['rdn'] = self.rdn
+            try:
+                if operation in cmapping['operation']:
+                    if eval(str(cmapping['condition'])):
+                        possible_dns.append(cmapping['rdn'] + '=' + str(data[cmapping['rdn']]) + ',' + cmapping['target'])
+            except KeyError, e:
+                import pdb; pdb.set_trace()
+                raise LoaderException('The key given in condition does not exist in record.(%s)' % str(e))
+        if len(possible_dns) == 0:
+            logging.warn('No container mapping condition matched for record %s' % str(data))
+            return None
         else:
-            raise LoaderException('The identifiers for LDAP must include the dn or nothing but not anything else.')
+            if len(possible_dns) >= 2: logging.warn('Found multiple matching container mapping conditions. Using the first one that matched')
+            return possible_dns[0]
 
     def _prepare_data(self, data):
         '''Prepares dn and data-dictionary for _update, _insert and _delete'''

hshetl/test/unit/__init__.py

@@ -34,4 +34,8 @@ fixtures = {
         [{'sn' : 'ipsum', 'cn' : 'erat', 'mail' : 'Suspendisse.aliquet.sem@aliquetmolestie.com', 'telephoneNumber' : '2456'},
          {'sn' : 'velit', 'cn' : 'sagittis', 'mail' : 'Vivamus.rhoncus.Donec@molestiedapibus.co.uk', 'telephoneNumber' : '1234'},
          {'sn' : 'velitt', 'cn' : 'foo', 'mail' : 'varius.et.euismod@ipsum.ca', 'telephoneNumber' : '5678'}],
+    'ldap_loader_container_mapping':
+        [{'operation': ['update'], 'condition': "'123' in data['telephoneNumber']", 'rdn': 'cn', 'target': 'foo=existent'},
+         {'operation': ['delete'], 'condition': "data['telephoneNumber'] == '5678'", 'rdn': 'cn', 'target': 'foo=trash'},
+         {'operation': ['insert'], 'condition': "True", 'rdn': 'sn', 'target': 'foo=new'}],
 }

hshetl/test/unit/test_loaders.py

@@ -212,7 +212,7 @@ class TestLdapLoader(unittest.TestCase):
     def setUp(self):
         self.ldap_connector = Mock(spec = connectors.LdapConnector)
         self.ldap_connector.name = 'FooConnector'
-        self.ldap_loader = loaders.LdapLoader(rdn = 'cn', base = 'foo=bar', objectClass = 'l', connector = self.ldap_connector)
+        self.ldap_loader = loaders.LdapLoader(rdn = 'cn', base = 'foo=bar', objectClass = 'l', connector = self.ldap_connector, condition_handling = 'DEFAULT')
         self.ldap_connection = Mock(spec = ldap.ldapobject.SimpleLDAPObject)
         self.ldap_connector.connection = self.ldap_connection
         self.ldap_connector.__enter__ = Mock(return_value = self.ldap_connector.connection)
@@ -234,6 +234,7 @@ class TestLdapLoader(unittest.TestCase):
 
     def test_update(self):
         c = 0
+        self.ldap_loader.container_mapping = [{'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': 'sn', 'target': 'foo=bar'}]
         for rec in self.result.update:
             rec.get_container_identifier = Mock(return_value = {'dn': 'sn=' + test.fixtures['ldap_syntax_loader_test_loading_old_data'][c]['sn'] + ',foo=bar'})
             rec.to_dict = Mock(side_effect = [test.fixtures['ldap_syntax_loader_test_loading_old_data'][c],
@@ -250,6 +251,7 @@ class TestLdapLoader(unittest.TestCase):
 
     def test_insert(self):
         c = 0
+        self.ldap_loader.container_mapping = [{'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': 'cn', 'target': 'foo=bar'}]
         for rec in self.result.insert:
             rec.get_container_identifier = Mock(return_value = None)
             rec.to_dict = Mock(return_value = test.fixtures['ldap_syntax_loader_test_loading'][c])
@@ -265,6 +267,7 @@ class TestLdapLoader(unittest.TestCase):
 
     def test_delete(self):
         c = 0
+        self.ldap_loader.container_mapping = [{'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': 'sn', 'target': 'foo=bar'}]
         for rec in self.result.delete:
             rec.get_container_identifier = Mock(return_value = {'dn': 'sn=' + test.fixtures['ldap_syntax_loader_test_loading_old_data'][c]['sn'] + ',foo=bar'})
             rec.to_dict = Mock(return_value = test.fixtures['ldap_syntax_loader_test_loading'][c])
@@ -279,18 +282,31 @@ class TestLdapLoader(unittest.TestCase):
         self.ldap_connection.delete_s.assert_has_calls(ldap_calls)
 
     def test_prepare_data_converts_everything_in_dataset_to_unicode(self):
-        self.ldap_loader.rdn = 'foo'
-        self.ldap_loader.base = 'foo=bar,poo=par'
+        self.ldap_loader.container_mapping = [{'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': 'foo', 'target': 'foo=bar,poo=par'}]
         self.ldap_loader.objectClass = 'virtualPersonStuffThing'
-        entity_dict = self.ldap_loader._prepare_data({'key' : 'value', 'foo' : 1337, 'bar' : True})
-        dn = self.ldap_loader._get_dn({'dn': 'someDN'}, {'key' : 'value', 'foo' : 1337, 'bar' : True})
-        self.assertEqual(dn, 'someDN')
-        self.assertEqual(entity_dict, {'objectClass' : 'virtualPersonStuffThing', 'key' : 'value', 'foo' : '1337', 'bar' : 'True'})
         entity_dict = self.ldap_loader._prepare_data({'key' : '!value', 'foo' : 42, 'bar' : False})
-        dn = self.ldap_loader._get_dn({'dn': 'foo=42,foo=bar,poo=par'}, {'key' : '!value', 'foo' : 42, 'bar' : False})
+        dn = self.ldap_loader._get_dn({'key' : '!value', 'foo' : 42, 'bar' : False}, 'insert')
         self.assertEqual(dn, 'foo=42,foo=bar,poo=par')
         self.assertEqual(entity_dict, {'objectClass' : 'virtualPersonStuffThing', 'key' : '!value', 'foo' : '42', 'bar' : 'False'})
 
+    def test_get_dn_resolves_container_mapping(self):
+        self.ldap_loader.container_mapping = test.fixtures['ldap_loader_container_mapping']
+        self.ldap_loader.container_mapping.append({'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': 'cn', 'target': 'foo=bar'})
+        records = test.fixtures['ldap_syntax_loader_test_loading']
+        dn = self.ldap_loader._get_dn(records[0], 'update')
+        self.assertEqual(dn, 'cn=erat,foo=existent')
+        dn = self.ldap_loader._get_dn(records[1], 'update')
+        self.assertEqual(dn, 'cn=sagittis,foo=existent')
+        dn = self.ldap_loader._get_dn(records[2], 'update')
+        self.assertEqual(dn, 'cn=nunc,foo=bar')
+        dn = self.ldap_loader._get_dn(records[2], 'delete')
+        self.assertEqual(dn, 'cn=nunc,foo=trash')
+        dn = self.ldap_loader._get_dn(records[1], 'insert')
+        self.assertEqual(dn, 'sn=velit,foo=new')
+        self.ldap_loader.container_mapping.remove({'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': 'cn', 'target': 'foo=bar'})
+        dn = self.ldap_loader._get_dn(records[2], 'update')
+        self.assertEqual(dn, None)
+
 
 if __name__ == "__main__":
     unittest.main()

hshetl/transformers.py

@@ -333,7 +333,7 @@ class UuidTransformer(AbstractTransformer):
         if isinstance(self.source, str) and self.entity is not None:
             self.source = NameResolver(self.entity, 'source', self.source)()
         else:
-            raise Exception("Source is of type string and entity is None, so the NameResolver cannot find a correspinding container.")
+            raise Exception("Source is of type string and entity is None, so the NameResolver cannot find a corresponding container.")
         if self.mapping is not {}:
             mapped_identifiers = [x if x not in self.mapping.keys() else self.mapping[x] for x in self.source.identifiers]
         self.result = Container(name = self.name,