Skip to content
Snippets Groups Projects
Commit b5fd907f authored by Dennis Ahrens's avatar Dennis Ahrens
Browse files

Revert "[TASK] Introduces container mapping for the ldap loader" 

This reverts commit e982a855.
And another related commit.
parent bd11064e
Branches
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
hshetl/loaders.py
+ 20
− 133
View file @ b5fd907f
......@@ -316,58 +316,20 @@ class SqlAlchemyLoader(AbstractLoader):
class LdapLoader(AbstractLoader):
'''A loader for LDAP data sources using the ldap and ldap.modlist modules.
The LDAP loader supports container mapping in the following form:
- condition_handling is a Flag which can either be LOG or DEFAULT
- on DEFAULT there will be a default container mapping created from the rdn and base given in init
- on LOG there will be NO default container mapping created which can result in records that could be loaded, because no container mapping could be applied(!), but you will get warnings in the log
- move is a list of actions that should move rather than do their actual task
- if "update" is in the list, the update action will move an entry AFTER the actual update of the entry
- if "delete" is in the list, the delete action will move an entry INSTED OF deleting it
- You can define as many container mapping as you want
- container mappings are priorised by order! Which means that the first container mapping that matches, will be applied!
- container mappings consist of operation, condition, target and an optional rdn
- the container mapping "operation" is a list of operations for which this container mapping should be applied
- the container mapping "condition" is python code, that will be thrown into eval (you can use the record object, unmapped property names in the data dict or the source(!) "dn" in the condition)
- the container mapping "target" is the base, where the record with the matching condition should be written or moved to
- the container mapping "rdn" is the rdn of the record, that will be the first part of the dn
- examples:
- condition:
- data['fooUnmapped'] == 'bar'
- True
- 'bar' in data['fooUnmapped']
- dn.lower() == "cn=some,ou=thing,ou=random,dc=de"
- target + rdn:
- target: ou=foo,dc=de | rdn: sn | dn: sn=valueOfsnInRecord,ou=foo,dc=de
.. function:: __init__(rdn, objectClass[, base = None[, container_mapping = [][, condition_handling = 'LOG'[, move = [][, **kwargs]]]]])
.. function:: __init__(rdn, base, objectClass[, **kwargs])
Constructor
:param rdn: The attribute used to create a DN for new records.
:param objectClass: The objectClass for new records - use this instead of providing it inside your entity.
:param base: The container in which new records will be created created.
:param container_mapping: The container mapping to be applied (priorised!!!)
:param condition_handling: Flag to determine, whether there should be a default mapping or not.
:param move: Determines, whether a record should be moved on update or delete, if a container mapping matches and WOULD move the record
:param objectClass: The objectClass for new records - use this instead of providing it inside your entity.
:param **kwargs: Accepts parameters from :class:`.AbstractExtractor`.
:type rdn: string
:type objectClass: string
:type base: string
:type container_mapping: list
:type condition_handling: string
:type move: list
:type objectClass: string
:type connector: :class:`hshetl.connectors.LdapConnector`
YAML definition samples:
Deprecated:
YAML definition sample:
.. code-block:: yaml
......@@ -377,39 +339,12 @@ class LdapLoader(AbstractLoader):
base: ou=rz,o=fh-hannover
objectClass: person
New:
.. code-block:: yaml
!ldaploader
connector: edirectory
rdn: cn
base: ou=rz,o=fh-hannover (optional if you have a condition mapping with the condition default)
objectClass: person
condition_handling: LOG (LOG or DEFAULT to create a default container mapping from the base)
move: ['update', 'delete']
container_mapping:
-
operation: ['insert', 'update', 'delete']
condition: data['l'] == 'Foo'
target: ou=foo,o=fh-hannover
rdn: sn (optional)
-
operation: ['insert']
condition: True
target: ou=new,o=fh-hannover
-
operation: ['update']
condition: dn.lower() == "cn=some,ou=thing,ou=random,dc=de"
target: ou=new,o=fh-hannover
'''
yaml_tag = u'!ldaploader'
'''Use this tag inside your YAML configuration, to define this loader.'''
#TODO: remove base once conversion to transformers is complete
def __init__(self, rdn, objectClass, base = None, container_mapping = [], condition_handling = 'LOG', move = [], **kwargs):
def __init__(self, rdn, base, objectClass, **kwargs):
super(LdapLoader, self).__init__(**kwargs)
self.rdn = rdn
'''The attribute that is used to create new DNs.'''
......@@ -417,14 +352,6 @@ class LdapLoader(AbstractLoader):
'''The container new records will be inserted in.'''
self.objectClass = objectClass
'''The objectClass of new inserted records.'''
self.container_mapping = container_mapping
if None in (self.rdn, self.objectClass) or (self.base == None and self.container_mapping[0]['target'] == None):
raise ConfigurationException('base, rdn or objectClass not found - needs to be configured in the connector or loader!')
if condition_handling == 'DEFAULT' and {'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': self.rdn, 'target': self.base} not in container_mapping:
self.container_mapping.append({'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': self.rdn, 'target': self.base})
'''The container mappings for this loader'''
self.move = move
'''Move records or not'''
def can_execute(self, connector):
'''Defines which connector can be handled by this extractor.'''
......@@ -433,6 +360,8 @@ class LdapLoader(AbstractLoader):
def _execute(self, result):
'''Executes the loading of data. Distinguishes between update, insert and delete.'''
logging.debug('Loads data: ' + self.__class__.__name__)
if None in (self.rdn, self.base, self.objectClass):
raise ConfigurationException('base, rdn or objectClass not found - needs to be configured in the connector or loader!')
with self.connector as connection:
for action in self.operations:
getattr(self, '_' + action)(connection, getattr(result, action))
......@@ -447,23 +376,16 @@ class LdapLoader(AbstractLoader):
'''Updates every dataset one after another.'''
for record in data:
old_entity_dict = self._prepare_data(record.to_dict(container = record.last_container.target_container))
entity_dict = self._prepare_data(record.to_dict())
dn = self._get_dn(record, 'update')
if dn == None: continue
old_dn = record.get_container_identifier()['dn'] if record.get_container_identifier() is not None else self.rdn + '=' + str(data[self.rdn]) + ',' + self.base
dict_data = record.to_dict()
entity_dict = self._prepare_data(dict_data)
dn = self._get_dn(record.get_container_identifier(), dict_data)
attributes = modlist.modifyModlist(old_entity_dict, entity_dict)
try:
if self.dry:
logging.info('Would modify dn {} with {} '.format(old_dn, self._generate_modify_logging(attributes, old_entity_dict, entity_dict)))
logging.info('Would modify dn {} with {} '.format(dn, self._generate_modify_logging(attributes, old_entity_dict, entity_dict)))
else:
connection.modify_s(dn, attributes)
logging.info('Modified dn {} with {} '.format(old_dn, self._generate_modify_logging(attributes, old_entity_dict, entity_dict)))
if dn != old_dn:
if 'update' in self.move:
logging.info('Container %s will be moved to %s after update' % (old_dn, dn))
self._move(connection, old_dn, dn)
else:
logging.info('Move after update prohibited by config (use move: [\'update\'])')
logging.info('Modified dn {} with {} '.format(dn, self._generate_modify_logging(attributes, old_entity_dict, entity_dict)))
except Exception, e:
logging.warn('Update failed for dn \'' + dn + '\': ' + str(e))
......@@ -471,8 +393,7 @@ class LdapLoader(AbstractLoader):
'''Inserts every dataset one after another.'''
for record in data:
entity_dict = self._prepare_data(record.to_dict())
dn = self._get_dn(record, 'insert')
if dn == None: continue
dn = self._get_dn(record.get_container_identifier(), record.to_dict())
attributes = modlist.addModlist(entity_dict)
try:
if self.dry:
......@@ -486,15 +407,7 @@ class LdapLoader(AbstractLoader):
def _delete(self, connection, data):
'''Deletes every entry one after another.'''
for record in data:
dn = self._get_dn(record, 'delete')
if dn == None: continue
old_dn = record.get_container_identifier()['dn'] if record.get_container_identifier() is not None else self.rdn + '=' + str(data[self.rdn]) + ',' + self.base
if dn != old_dn:
if 'delete' in self.move:
logging.info('Container %s will be moved to %s instead of being deleted' % (old_dn, dn))
self._move(connection, old_dn, dn)
else:
logging.info('Move after delete prohibited by config (use move: [\'delete\'])')
dn = self._get_dn(record.get_container_identifier(), record.to_dict())
try:
if self.dry:
logging.info('Would delete dn {}'.format(dn))
......@@ -504,39 +417,13 @@ class LdapLoader(AbstractLoader):
except Exception, e:
logging.warn('Delete failed for dn \'' + dn + '\': ' + str(e))
def _move(self, connection, old_dn, new_dn):
'''Moves a single record'''
try:
if self.dry:
logging.info('Would move dn %s to %s' % (old_dn, new_dn))
else:
rdn = new_dn.split(',')[0]
newsuperior = new_dn[len(rdn)+1:]
connection.rename_s(old_dn, rdn, newsuperior)
logging.info('Moved dn %s to %s' % (old_dn, new_dn))
except Exception, e:
logging.warn('Move failed for old dn %s to new dn %s' % (old_dn, new_dn))
def _get_dn(self, record, operation):
'''Resolves the dn for the given record'''
possible_dns = []
data = record.to_dict(record.last_container.target_container) if operation == 'delete' else record.to_dict()
dn = record.get_container_identifier()['dn'] if operation in ['update', 'delete'] else None
for cmapping in self.container_mapping:
if not cmapping.has_key('rdn'):
cmapping['rdn'] = self.rdn
try:
if operation in cmapping['operation']:
if eval(str(cmapping['condition'])):
possible_dns.append(cmapping['rdn'] + '=' + str(data[cmapping['rdn']]) + ',' + cmapping['target'])
except KeyError, e:
raise LoaderException('The key given in condition does not exist in record.(%s)' % str(e))
if len(possible_dns) == 0:
logging.warn('No container mapping condition matched for record %s' % str(data))
return None
def _get_dn(self, identifier, data):
if isinstance(identifier, dict) and 'dn' in identifier and len(identifier) == 1:
return identifier['dn']
elif identifier == None:
return self.rdn + '=' + str(data[self.rdn]) + ',' + self.base
else:
if len(possible_dns) >= 2: logging.warn('Found multiple matching container mapping conditions. Using the first one that matched')
return possible_dns[0]
raise LoaderException('The identifiers for LDAP must include the dn or nothing but not anything else.')
def _prepare_data(self, data):
'''Prepares dn and data-dictionary for _update, _insert and _delete'''
......
hshetl/test/unit/__init__.py
+ 0
− 5
View file @ b5fd907f
......@@ -34,9 +34,4 @@ fixtures = {
[{'sn' : 'ipsum', 'cn' : 'erat', 'mail' : 'Suspendisse.aliquet.sem@aliquetmolestie.com', 'telephoneNumber' : '2456'},
{'sn' : 'velit', 'cn' : 'sagittis', 'mail' : 'Vivamus.rhoncus.Donec@molestiedapibus.co.uk', 'telephoneNumber' : '1234'},
{'sn' : 'velitt', 'cn' : 'foo', 'mail' : 'varius.et.euismod@ipsum.ca', 'telephoneNumber' : '5678'}],
'ldap_loader_container_mapping':
[{'operation': ['update'], 'condition': "'123' in data['telephoneNumber']", 'rdn': 'cn', 'target': 'foo=existent'},
{'operation': ['delete'], 'condition': "data['telephoneNumber'] == '5678'", 'rdn': 'cn', 'target': 'foo=trash'},
{'operation': ['insert'], 'condition': "True", 'rdn': 'sn', 'target': 'foo=new'},
{'operation': ['update'], 'condition': "dn == 'cn=nunc,foo=bar'", 'rdn': 'cn', 'target': 'foo=move'}],
}
hshetl/test/unit/test_loaders.py
+ 9
− 36
View file @ b5fd907f
......@@ -212,7 +212,7 @@ class TestLdapLoader(unittest.TestCase):
def setUp(self):
self.ldap_connector = Mock(spec = connectors.LdapConnector)
self.ldap_connector.name = 'FooConnector'
self.ldap_loader = loaders.LdapLoader(rdn = 'cn', base = 'foo=bar', objectClass = 'l', connector = self.ldap_connector, condition_handling = 'DEFAULT')
self.ldap_loader = loaders.LdapLoader(rdn = 'cn', base = 'foo=bar', objectClass = 'l', connector = self.ldap_connector)
self.ldap_connection = Mock(spec = ldap.ldapobject.SimpleLDAPObject)
self.ldap_connector.connection = self.ldap_connection
self.ldap_connector.__enter__ = Mock(return_value = self.ldap_connector.connection)
......@@ -234,12 +234,10 @@ class TestLdapLoader(unittest.TestCase):
def test_update(self):
c = 0
self.ldap_loader.container_mapping = [{'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': 'sn', 'target': 'foo=bar'}]
for rec in self.result.update:
rec.get_container_identifier = Mock(return_value = {'dn': 'sn=' + test.fixtures['ldap_syntax_loader_test_loading_old_data'][c]['sn'] + ',foo=bar'})
rec.to_dict = Mock(side_effect = [test.fixtures['ldap_syntax_loader_test_loading_old_data'][c],
test.fixtures['ldap_syntax_loader_test_loading'][c],
test.fixtures['ldap_syntax_loader_test_loading_old_data'][c]])
test.fixtures['ldap_syntax_loader_test_loading'][c]])
rec.last_container = Mock()
rec.last_container.target_container = Mock()
c += 1
......@@ -252,7 +250,6 @@ class TestLdapLoader(unittest.TestCase):
def test_insert(self):
c = 0
self.ldap_loader.container_mapping = [{'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': 'cn', 'target': 'foo=bar'}]
for rec in self.result.insert:
rec.get_container_identifier = Mock(return_value = None)
rec.to_dict = Mock(return_value = test.fixtures['ldap_syntax_loader_test_loading'][c])
......@@ -268,7 +265,6 @@ class TestLdapLoader(unittest.TestCase):
def test_delete(self):
c = 0
self.ldap_loader.container_mapping = [{'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': 'sn', 'target': 'foo=bar'}]
for rec in self.result.delete:
rec.get_container_identifier = Mock(return_value = {'dn': 'sn=' + test.fixtures['ldap_syntax_loader_test_loading_old_data'][c]['sn'] + ',foo=bar'})
rec.to_dict = Mock(return_value = test.fixtures['ldap_syntax_loader_test_loading'][c])
......@@ -283,41 +279,18 @@ class TestLdapLoader(unittest.TestCase):
self.ldap_connection.delete_s.assert_has_calls(ldap_calls)
def test_prepare_data_converts_everything_in_dataset_to_unicode(self):
self.ldap_loader.container_mapping = [{'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': 'foo', 'target': 'foo=bar,poo=par'}]
self.ldap_loader.rdn = 'foo'
self.ldap_loader.base = 'foo=bar,poo=par'
self.ldap_loader.objectClass = 'virtualPersonStuffThing'
entity_dict = self.ldap_loader._prepare_data({'key' : 'value', 'foo' : 1337, 'bar' : True})
dn = self.ldap_loader._get_dn({'dn': 'someDN'}, {'key' : 'value', 'foo' : 1337, 'bar' : True})
self.assertEqual(dn, 'someDN')
self.assertEqual(entity_dict, {'objectClass' : 'virtualPersonStuffThing', 'key' : 'value', 'foo' : '1337', 'bar' : 'True'})
entity_dict = self.ldap_loader._prepare_data({'key' : '!value', 'foo' : 42, 'bar' : False})
record = self.result.insert[0]
record.to_dict = Mock(return_value = {'key' : '!value', 'foo' : 42, 'bar' : False})
record.get_container_identifier = Mock(return_value = None)
dn = self.ldap_loader._get_dn(record, 'insert')
dn = self.ldap_loader._get_dn({'dn': 'foo=42,foo=bar,poo=par'}, {'key' : '!value', 'foo' : 42, 'bar' : False})
self.assertEqual(dn, 'foo=42,foo=bar,poo=par')
self.assertEqual(entity_dict, {'objectClass' : 'virtualPersonStuffThing', 'key' : '!value', 'foo' : '42', 'bar' : 'False'})
def test_get_dn_resolves_container_mapping(self):
self.ldap_loader.container_mapping = test.fixtures['ldap_loader_container_mapping']
self.ldap_loader.container_mapping.append({'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': 'cn', 'target': 'foo=bar'})
records = self.result.update
c = 0
for rec in records:
rec.get_container_identifier = Mock(return_value = {'dn': 'cn=' + test.fixtures['ldap_syntax_loader_test_loading'][c]['cn'] + ',foo=bar'})
rec.to_dict = Mock(return_value = test.fixtures['ldap_syntax_loader_test_loading'][c])
rec.last_container = Mock()
rec.last_container.target_container = Mock()
c += 1
dn = self.ldap_loader._get_dn(records[0], 'update')
self.assertEqual(dn, 'cn=erat,foo=existent')
dn = self.ldap_loader._get_dn(records[1], 'update')
self.assertEqual(dn, 'cn=sagittis,foo=existent')
dn = self.ldap_loader._get_dn(records[2], 'update')
self.assertEqual(dn, 'cn=nunc,foo=move')
dn = self.ldap_loader._get_dn(records[2], 'delete')
self.assertEqual(dn, 'cn=nunc,foo=trash')
dn = self.ldap_loader._get_dn(records[1], 'insert')
self.assertEqual(dn, 'sn=velit,foo=new')
self.ldap_loader.container_mapping.remove({'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': 'cn', 'target': 'foo=bar'})
dn = self.ldap_loader._get_dn(records[0], 'delete')
self.assertEqual(dn, None)
if __name__ == "__main__":
unittest.main()
hshetl/transformers.py
+ 1
− 1
View file @ b5fd907f
......@@ -333,7 +333,7 @@ class UuidTransformer(AbstractTransformer):
if isinstance(self.source, str) and self.entity is not None:
self.source = NameResolver(self.entity, 'source', self.source)()
else:
raise Exception("Source is of type string and entity is None, so the NameResolver cannot find a corresponding container.")
raise Exception("Source is of type string and entity is None, so the NameResolver cannot find a correspinding container.")
if self.mapping is not {}:
mapped_identifiers = [x if x not in self.mapping.keys() else self.mapping[x] for x in self.source.identifiers]
self.result = Container(name = self.name,
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment