diff --git a/hshetl/loaders.py b/hshetl/loaders.py
index 3e9a7eeefa959ef8a649705dd8aa5590215dca99..bbf7ad7fa81c9ab92bbd683fdddd36c70bf02476 100644
--- a/hshetl/loaders.py
+++ b/hshetl/loaders.py
@@ -320,17 +320,18 @@ class LdapLoader(AbstractLoader):
- condition_handling is a Flag which can either be LOG or DEFAULT
- - on DEFAULT there will be a default container mapping created from the rdn and base give in init
+ - on DEFAULT there will be a default container mapping created from the rdn and base given in init
- on LOG there will be NO default container mapping created which can result in records that could be loaded, because no container mapping could be applied(!), but you will get warnings in the log
- move is a list of actions that should move rather than do their actual task
- if "update" is in the list, the update action will move an entry AFTER the actual update of the entry
- if "delete" is in the list, the delete action will move an entry INSTED OF deleting it
- You can define as many container mapping as you want
+ - container mappings are priorised by order! Which means that the first container mapping that matches, will be applied!
- container mappings consist of operation, condition, target and an optional rdn
- - the container mapping "operation" is a list of operations for which this containerm mapping should be applied
- - the container mapping "condition" is python code, that will be thrown into eval (you can ONLY use UNMAPPED property names in the condition!)
+ - the container mapping "operation" is a list of operations for which this container mapping should be applied
+ - the container mapping "condition" is python code, that will be thrown into eval (you can use the record object, unmapped property names in the data dict or the source(!) "dn" in the condition)
- the container mapping "target" is the base, where the record with the matching condition should be written or moved to
- the container mapping "rdn" is the rdn of the record, that will be the first part of the dn
- examples:
@@ -340,9 +341,10 @@ class LdapLoader(AbstractLoader):
- data['fooUnmapped'] == 'bar'
- True
- 'bar' in data['fooUnmapped']
+ - dn.lower() == "cn=some,ou=thing,ou=random,dc=de"
- target + rdn:
- - target: ou=foo,dc=de rdn: sn dn: sn=valueOfsnInRecord,ou=foo,dc=de
+ - target: ou=foo,dc=de | rdn: sn | dn: sn=valueOfsnInRecord,ou=foo,dc=de
.. function:: __init__(rdn, objectClass[, base = None[, container_mapping = [][, condition_handling = 'LOG'[, move = [][, **kwargs]]]]])
@@ -396,6 +398,10 @@ class LdapLoader(AbstractLoader):
operation: ['insert']
condition: True
target: ou=new,o=fh-hannover
+ -
+ operation: ['update']
+ condition: dn.lower() == "cn=some,ou=thing,ou=random,dc=de"
+ target: ou=new,o=fh-hannover
'''
@@ -441,18 +447,17 @@ class LdapLoader(AbstractLoader):
'''Updates every dataset one after another.'''
for record in data:
old_entity_dict = self._prepare_data(record.to_dict(container = record.last_container.target_container))
- dict_data = record.to_dict()
- entity_dict = self._prepare_data(dict_data)
- dn = self._get_dn(dict_data, 'update')
+ entity_dict = self._prepare_data(record.to_dict())
+ dn = self._get_dn(record, 'update')
if dn == None: continue
old_dn = record.get_container_identifier()['dn'] if record.get_container_identifier() is not None else self.rdn + '=' + str(data[self.rdn]) + ',' + self.base
attributes = modlist.modifyModlist(old_entity_dict, entity_dict)
try:
if self.dry:
- logging.info('Would modify dn {} with {} '.format(dn, self._generate_modify_logging(attributes, old_entity_dict, entity_dict)))
+ logging.info('Would modify dn {} with {} '.format(old_dn, self._generate_modify_logging(attributes, old_entity_dict, entity_dict)))
else:
connection.modify_s(dn, attributes)
- logging.info('Modified dn {} with {} '.format(dn, self._generate_modify_logging(attributes, old_entity_dict, entity_dict)))
+ logging.info('Modified dn {} with {} '.format(old_dn, self._generate_modify_logging(attributes, old_entity_dict, entity_dict)))
if dn != old_dn:
if 'update' in self.move:
logging.info('Container %s will be moved to %s after update' % (old_dn, dn))
@@ -466,7 +471,7 @@ class LdapLoader(AbstractLoader):
'''Inserts every dataset one after another.'''
for record in data:
entity_dict = self._prepare_data(record.to_dict())
- dn = self._get_dn(record.to_dict(), 'insert')
+ dn = self._get_dn(record, 'insert')
if dn == None: continue
attributes = modlist.addModlist(entity_dict)
try:
@@ -481,7 +486,7 @@ class LdapLoader(AbstractLoader):
def _delete(self, connection, data):
'''Deletes every entry one after another.'''
for record in data:
- dn = self._get_dn(record.to_dict(record.last_container.target_container), 'delete')
+ dn = self._get_dn(record, 'delete')
if dn == None: continue
old_dn = record.get_container_identifier()['dn'] if record.get_container_identifier() is not None else self.rdn + '=' + str(data[self.rdn]) + ',' + self.base
if dn != old_dn:
@@ -512,9 +517,11 @@ class LdapLoader(AbstractLoader):
except Exception, e:
logging.warn('Move failed for old dn %s to new dn %s' % (old_dn, new_dn))
- def _get_dn(self, data, operation):
+ def _get_dn(self, record, operation):
'''Resolves the dn for the given record'''
possible_dns = []
+ data = record.to_dict(record.last_container.target_container) if operation == 'delete' else record.to_dict()
+ dn = record.get_container_identifier()['dn'] if operation in ['update', 'delete'] else None
for cmapping in self.container_mapping:
if not cmapping.has_key('rdn'):
cmapping['rdn'] = self.rdn
@@ -523,7 +530,6 @@ class LdapLoader(AbstractLoader):
if eval(str(cmapping['condition'])):
possible_dns.append(cmapping['rdn'] + '=' + str(data[cmapping['rdn']]) + ',' + cmapping['target'])
except KeyError, e:
- import pdb; pdb.set_trace()
raise LoaderException('The key given in condition does not exist in record.(%s)' % str(e))
if len(possible_dns) == 0:
logging.warn('No container mapping condition matched for record %s' % str(data))
diff --git a/hshetl/test/unit/__init__.py b/hshetl/test/unit/__init__.py
index 627b1cc14297fa0fb20639e1fb74bcb766c49ad4..44cb0dac534a886d9ac542b3df74ac3329bbbffe 100644
--- a/hshetl/test/unit/__init__.py
+++ b/hshetl/test/unit/__init__.py
@@ -37,5 +37,6 @@ fixtures = {
'ldap_loader_container_mapping':
[{'operation': ['update'], 'condition': "'123' in data['telephoneNumber']", 'rdn': 'cn', 'target': 'foo=existent'},
{'operation': ['delete'], 'condition': "data['telephoneNumber'] == '5678'", 'rdn': 'cn', 'target': 'foo=trash'},
- {'operation': ['insert'], 'condition': "True", 'rdn': 'sn', 'target': 'foo=new'}],
+ {'operation': ['insert'], 'condition': "True", 'rdn': 'sn', 'target': 'foo=new'},
+ {'operation': ['update'], 'condition': "dn == 'cn=nunc,foo=bar'", 'rdn': 'cn', 'target': 'foo=move'}],
}
diff --git a/hshetl/test/unit/test_loaders.py b/hshetl/test/unit/test_loaders.py
index e97f64a067720120afef34a393b040e540a5df02..a12a98df031df233a3b6df1e1dacea8368b91557 100644
--- a/hshetl/test/unit/test_loaders.py
+++ b/hshetl/test/unit/test_loaders.py
@@ -238,7 +238,8 @@ class TestLdapLoader(unittest.TestCase):
for rec in self.result.update:
rec.get_container_identifier = Mock(return_value = {'dn': 'sn=' + test.fixtures['ldap_syntax_loader_test_loading_old_data'][c]['sn'] + ',foo=bar'})
rec.to_dict = Mock(side_effect = [test.fixtures['ldap_syntax_loader_test_loading_old_data'][c],
- test.fixtures['ldap_syntax_loader_test_loading'][c]])
+ test.fixtures['ldap_syntax_loader_test_loading'][c],
+ test.fixtures['ldap_syntax_loader_test_loading_old_data'][c]])
rec.last_container = Mock()
rec.last_container.target_container = Mock()
c += 1
@@ -285,26 +286,36 @@ class TestLdapLoader(unittest.TestCase):
self.ldap_loader.container_mapping = [{'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': 'foo', 'target': 'foo=bar,poo=par'}]
self.ldap_loader.objectClass = 'virtualPersonStuffThing'
entity_dict = self.ldap_loader._prepare_data({'key' : '!value', 'foo' : 42, 'bar' : False})
- dn = self.ldap_loader._get_dn({'key' : '!value', 'foo' : 42, 'bar' : False}, 'insert')
+ record = self.result.insert[0]
+ record.to_dict = Mock(return_value = {'key' : '!value', 'foo' : 42, 'bar' : False})
+ record.get_container_identifier = Mock(return_value = None)
+ dn = self.ldap_loader._get_dn(record, 'insert')
self.assertEqual(dn, 'foo=42,foo=bar,poo=par')
self.assertEqual(entity_dict, {'objectClass' : 'virtualPersonStuffThing', 'key' : '!value', 'foo' : '42', 'bar' : 'False'})
def test_get_dn_resolves_container_mapping(self):
self.ldap_loader.container_mapping = test.fixtures['ldap_loader_container_mapping']
self.ldap_loader.container_mapping.append({'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': 'cn', 'target': 'foo=bar'})
- records = test.fixtures['ldap_syntax_loader_test_loading']
+ records = self.result.update
+ c = 0
+ for rec in records:
+ rec.get_container_identifier = Mock(return_value = {'dn': 'cn=' + test.fixtures['ldap_syntax_loader_test_loading'][c]['cn'] + ',foo=bar'})
+ rec.to_dict = Mock(return_value = test.fixtures['ldap_syntax_loader_test_loading'][c])
+ rec.last_container = Mock()
+ rec.last_container.target_container = Mock()
+ c += 1
dn = self.ldap_loader._get_dn(records[0], 'update')
self.assertEqual(dn, 'cn=erat,foo=existent')
dn = self.ldap_loader._get_dn(records[1], 'update')
self.assertEqual(dn, 'cn=sagittis,foo=existent')
dn = self.ldap_loader._get_dn(records[2], 'update')
- self.assertEqual(dn, 'cn=nunc,foo=bar')
+ self.assertEqual(dn, 'cn=nunc,foo=move')
dn = self.ldap_loader._get_dn(records[2], 'delete')
self.assertEqual(dn, 'cn=nunc,foo=trash')
dn = self.ldap_loader._get_dn(records[1], 'insert')
self.assertEqual(dn, 'sn=velit,foo=new')
self.ldap_loader.container_mapping.remove({'operation': ['insert', 'update', 'delete'],'condition': True, 'rdn': 'cn', 'target': 'foo=bar'})
- dn = self.ldap_loader._get_dn(records[2], 'update')
+ dn = self.ldap_loader._get_dn(records[0], 'delete')
self.assertEqual(dn, None)