Skip to content
Snippets Groups Projects
Commit 37813797 authored by Dennis Ahrens's avatar Dennis Ahrens
Browse files

[TASK] Fixes LDAP issues

Fixes the option handling of connections and
encoding issue when extracting from LDAP.
parent 4c08a805
No related branches found
No related tags found
No related merge requests found
...@@ -289,7 +289,7 @@ class Dialect(csv.Dialect): ...@@ -289,7 +289,7 @@ class Dialect(csv.Dialect):
:type delimiter: string :type delimiter: string
:type doublequote: bool :type doublequote: bool
:type escapechar: string or False :type escapechar: mixed
:type lineterminator: string :type lineterminator: string
:type quotechar: string :type quotechar: string
:type skipinitialspace: bool :type skipinitialspace: bool
...@@ -316,7 +316,7 @@ class Dialect(csv.Dialect): ...@@ -316,7 +316,7 @@ class Dialect(csv.Dialect):
def __init__(self, def __init__(self,
delimiter = ',', delimiter = ',',
doublequote = True, doublequote = True,
escapechar = False, escapechar = None,
lineterminator = '\n', lineterminator = '\n',
quotechar = '"', quotechar = '"',
skipinitialspace = False, skipinitialspace = False,
......
...@@ -193,7 +193,7 @@ class LdapConnector(AbstractConnector): ...@@ -193,7 +193,7 @@ class LdapConnector(AbstractConnector):
yaml_tag = u'!ldap' yaml_tag = u'!ldap'
'''Use this tag inside your YAML configuration, to define this connector.''' '''Use this tag inside your YAML configuration, to define this connector.'''
def __init__(self, uri, base = None, scope = 'subtree', page_size = False, start_tls = True, **kwargs): def __init__(self, uri, base = None, scope = 'subtree', page_size = False, start_tls = True, options = {}, **kwargs):
'''Initializes the LdapStorageInterface. '''Initializes the LdapStorageInterface.
Internally creates a storage_interface with the ldap module. Internally creates a storage_interface with the ldap module.
...@@ -218,11 +218,7 @@ class LdapConnector(AbstractConnector): ...@@ -218,11 +218,7 @@ class LdapConnector(AbstractConnector):
self.security_level = 'none' self.security_level = 'none'
else: else:
self.security_level = 'tls' self.security_level = 'tls'
self.options = {} self.options = options
for k, v in kwargs.iteritems():
if k in ('name', 'encoding'): continue
self.options[getattr(ldap, k)] = v
del kwargs[k]
super(LdapConnector, self).__init__(uri = uri, **kwargs) super(LdapConnector, self).__init__(uri = uri, **kwargs)
def _connect(self): def _connect(self):
...@@ -230,7 +226,7 @@ class LdapConnector(AbstractConnector): ...@@ -230,7 +226,7 @@ class LdapConnector(AbstractConnector):
self.connection = ldap.initialize(self.uri) self.connection = ldap.initialize(self.uri)
for option, value in self.options.iteritems(): for option, value in self.options.iteritems():
logging.debug('Set option ' + repr(option) + ' = ' + repr(value)) logging.debug('Set option ' + repr(option) + ' = ' + repr(value))
self.connection.set_option(option, value) self.connection.set_option(getattr(ldap, option), value)
if self.security_level == 'tls': if self.security_level == 'tls':
logging.debug('Connecting with TLS') logging.debug('Connecting with TLS')
self.connection.start_tls_s() self.connection.start_tls_s()
......
...@@ -287,10 +287,11 @@ class PropertyConverter(object): ...@@ -287,10 +287,11 @@ class PropertyConverter(object):
encoding = self._detect_encoding(value) encoding = self._detect_encoding(value)
logging.debug('Detected raw value ' + repr(value) + 'with encoding: ' + encoding) logging.debug('Detected raw value ' + repr(value) + 'with encoding: ' + encoding)
if encoding not in ('Unknown', 'utf-8'): if encoding not in ('Unknown', 'utf-8'):
logging.debug('decode... encode')
raw_value = value.decode(encoding) raw_value = value.decode(encoding)
return unicode(raw_value) return unicode(raw_value)
elif encoding == 'utf-8': elif encoding == 'utf-8':
return unicode(value) return unicode(value, 'utf-8')
else: else:
raise ValueError('hshetl - Encoding trouble.') raise ValueError('hshetl - Encoding trouble.')
......
...@@ -204,7 +204,9 @@ class LdapExtractor(AbstractExtractor): ...@@ -204,7 +204,9 @@ class LdapExtractor(AbstractExtractor):
if dn == None: continue if dn == None: continue
row = {'dn': dn} row = {'dn': dn}
for key, value in entry.items(): for key, value in entry.items():
if len(value) is 1: value = value[0].decode(self.connector.encoding) logging.debug('%s %s' % (key, value))
value = map(lambda v: unicode(v.decode(self.connector.encoding)), value)
if len(value) is 1: value = value[0]
row[key] = value row[key] = value
result.append(row) result.append(row)
return result return result
......
...@@ -206,6 +206,7 @@ class SqlAlchemyLoader(AbstractLoader): ...@@ -206,6 +206,7 @@ class SqlAlchemyLoader(AbstractLoader):
'''Use this tag inside your YAML configuration, to define this loader.''' '''Use this tag inside your YAML configuration, to define this loader.'''
def __init__(self, table_name, **kwargs): def __init__(self, table_name, **kwargs):
logging.debug('SqlAlchemyLoader constructor arguments: %s' % kwargs)
super(SqlAlchemyLoader, self).__init__(**kwargs) super(SqlAlchemyLoader, self).__init__(**kwargs)
self.table_name = table_name self.table_name = table_name
'''The name of the table where the records will be load.''' '''The name of the table where the records will be load.'''
...@@ -216,6 +217,7 @@ class SqlAlchemyLoader(AbstractLoader): ...@@ -216,6 +217,7 @@ class SqlAlchemyLoader(AbstractLoader):
def _execute(self, result): def _execute(self, result):
'''Executes the loading of data. Distinguishes between update, insert and delete''' '''Executes the loading of data. Distinguishes between update, insert and delete'''
logging.debug('Will execute the following operations: %s' % self.operations)
self.table = Table(self.table_name, self.table = Table(self.table_name,
MetaData(), MetaData(),
autoload_with = self.connector.engine, autoload_with = self.connector.engine,
...@@ -224,9 +226,12 @@ class SqlAlchemyLoader(AbstractLoader): ...@@ -224,9 +226,12 @@ class SqlAlchemyLoader(AbstractLoader):
with self.connector as connection: with self.connector as connection:
for action in self.operations: for action in self.operations:
getattr(self, '_' + action)(connection, getattr(result, action)) getattr(self, '_' + action)(connection, getattr(result, action))
logging.info(str(len(result.insert)) + ' data sets inserted, ' logging.info('%s data sets inserted (%s), %s data sets updated (%s) , %s data sets deleted (%s).' % (str(len(result.insert)),
+ str(len(result.update)) + ' data sets updated, ' 'Done!' if 'insert' in self.operations else 'Skipped!',
+ str(len(result.delete)) + ' data sets deleted.') str(len(result.update)),
'Done!' if 'update' in self.operations else 'Skipped!',
str(len(result.delete)),
'Done!' if 'delete' in self.operations else 'Skipped!'))
def _insert(self, connection, data): def _insert(self, connection, data):
'''Creates a sql insert statement using sqlalchemy and executes it on the connection.''' '''Creates a sql insert statement using sqlalchemy and executes it on the connection.'''
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment