[TASK] Better logging messages.

For productive usage the logs should give more
information when running in the background without
using inspection.

hshetl/entities.py

@@ -90,7 +90,7 @@ class Entity(AbstractRepository):
         logging.info('Load {} records into the entity {}.'.format(len(data), self.name))
         for raw_record in data:
             self._load_record(raw_record, container)
-        logging.info('Load of {} records into the entity {} done with avarage load time.'.format(len(data), self.name))
+        logging.info('Load of {} records into the entity {} done.'.format(len(data), self.name))
 
     def _load_record(self, raw_record, container):
         '''Adds a record to the entity repository.'''
@@ -226,7 +226,7 @@ class Container(list):
             msg = 'Collision in %s system detected! Aborting! %s found twice.' % (self.name, repr(record))
             raise DuplicatedSystemIdException(msg)
         else:
-            msg = 'Collision in %s container detected! Ignoring collision. %s found twice and added to collision.' % (self.name, repr(record))
+            msg = 'Collision in %s container detected! Ignoring collision.\n%s' % (self.name, record)
             logging.warn(msg)
             self.collisions.append(record)
 
@@ -238,7 +238,7 @@ class Container(list):
             msg = 'Collision in %s system detected! Aborting! %s found twice.' % (self.name, repr(record))
             raise DuplicatedJoinIdException(msg)
         else:
-            msg = 'Collision in %s container detected! Ignoring collision. %s found twice and added to collision.' % (self.name, repr(record))
+            msg = 'Collision in %s container detected! Ignoring collision.\n%s' % (self.name, record)
             logging.warn(msg)
             self.join_collisions.append(record)
 
@@ -498,10 +498,13 @@ class Record(object):
         return result
 
     def __unicode__(self):
-        o = 'Record:\n'
+        o = u'<Record:'
+        containers = []
         for name, container in self.containers.iteritems():
-            o += '{}: id: {} properties: {}\n'.format(name, str(self.get_container_identifier(container)), str(self.to_dict(container = container, remove_system_identifiers = False)))
-        return o
+            container_id = self.get_container_identifier(container)
+            if container_id != None:
+                containers.append(u'container: {}: id: {} properties: {}'.format(name, str(container_id), str(self.to_dict(container = container, remove_system_identifiers = False))))
+        return o + u'\n'.join(containers) + u'>'
 
 
 class EntityRepository(AbstractRepository):

hshetl/extractors.py

@@ -85,6 +85,9 @@ class AbstractExtractor(object):
         '''Executes the extraction of data. Implementation specific.'''
         raise NotImplementedError
 
+    def __unicode__(self):
+        return u'<AbstractExtractor with connector: %s>' % self.connector.name
+
 
 @yamlify
 class SqlAlchemyExtractor(AbstractExtractor):
@@ -136,6 +139,9 @@ class SqlAlchemyExtractor(AbstractExtractor):
             return 'SELECT ' + query['select'] + ' FROM ' + query['from'] + ' WHERE ' + query['where']
         raise ConfigurationException('The query must be a string or a dictionary containing an key sql that has a string value or a dict containing the keys select, from and where. Encountered {}'.format(type(query)))
 
+    def __unicode__(self):
+        return u'<SqlAlchemyExtractor with connector: %s, query: %s>' % (self.connector.name, self.query)
+
 
 @yamlify
 class LdapExtractor(AbstractExtractor):
@@ -276,6 +282,9 @@ class LdapExtractor(AbstractExtractor):
                 else: raise ExtractorException('Warning:  LdapServer ignores RFC 2696 control.')
         return raw_result
 
+    def __unicode__(self):
+        return u'<LdapExtractor with connector: %s, base: %s, scope: %s, filter: %s>' % (((self.connector.name,) + self._merge_connector_configuration())[:-1] + (self.ldap_filter,))
+
 
 @yamlify
 class CsvExtractor(AbstractExtractor):

hshetl/jobs.py

@@ -671,6 +671,7 @@ class LoadJob(EntityJob, ConnectorJob):
     def _execute(self):
         '''Load data from the repository into a storage interface.'''
         result = Result(entity = self.entity, source = self.source)
+        logging.info(unicode(result))
         self.loader.execute(result)
 
 

hshetl/loaders.py

@@ -239,7 +239,7 @@ class SqlAlchemyLoader(AbstractLoader):
         for record in data:
             values.append(record.to_dict(remove_system_identifiers = False))
         statement = self.table.insert()
-        logging.debug('Executing {} with {} '.format(str(statement), str(values)))
+        logging.info('Executing {} with {} '.format(str(statement), str(values)))
         connection.execute(statement, values)
 
     def _update(self, connection, data):
@@ -247,7 +247,7 @@ class SqlAlchemyLoader(AbstractLoader):
         for record in data:
             statement = self.table.update(values = record.to_dict())
             statement = self._add_where_constraints(statement, record)
-            logging.debug('executing sql update query: ' + str(statement))
+            logging.info('Executing sql update query: ' + str(statement))
             connection.execute(statement)
 
 
@@ -256,7 +256,7 @@ class SqlAlchemyLoader(AbstractLoader):
         for record in data:
             statement = self.table.delete()
             statement = self._add_where_constraints(statement, record)
-            logging.debug('executing sql delete query: ' + str(statement))
+            logging.info('Executing sql delete query: ' + str(statement))
             connection.execute(statement)
 
     def _add_where_constraints(self, statement, record):
@@ -330,6 +330,12 @@ class LdapLoader(AbstractLoader):
         with self.connector as connection:
             for action in self.operations:
                 getattr(self, '_' + action)(connection, getattr(result, action))
+        logging.info('%s data sets inserted (%s), %s data sets updated (%s) , %s data sets deleted (%s).' % (str(len(result.insert)),
+                                                                                                       'Done!' if 'insert' in self.operations else 'Skipped!',
+                                                                                                       str(len(result.update)),
+                                                                                                       'Done!' if 'update' in self.operations else 'Skipped!',
+                                                                                                       str(len(result.delete)),
+                                                                                                       'Done!' if 'delete' in self.operations else 'Skipped!'))
 
     def _update(self, connection, data):
         '''Updates every dataset one after another.'''
@@ -341,6 +347,7 @@ class LdapLoader(AbstractLoader):
             attributes = modlist.modifyModlist(old_entity_dict, entity_dict)
             try:
                 connection.modify_s(dn, attributes)
+                logging.info('Modified dn {} with {} '.format(dn, str(attributes)))
             except Exception, e:
                 logging.warn('Update failed for dn \'' + dn + '\': ' + str(e))
 
@@ -352,6 +359,7 @@ class LdapLoader(AbstractLoader):
             attributes = modlist.addModlist(entity_dict)
             try:
                 connection.add_s(dn, attributes)
+                logging.info('Created dn {} with {} '.format(dn, str(attributes)))
             except Exception, e:
                 logging.warn('Insert failed for dn \'' + dn + '\': ' + str(e))
 
@@ -361,6 +369,7 @@ class LdapLoader(AbstractLoader):
             dn = self._get_dn(record.get_container_identifier(), record.to_dict())
             try:
                 connection.delete_s(dn)
+                logging.info('Deleted dn {}'.format(dn))
             except Exception, e:
                 logging.warn('Delete failed for dn \'' + dn + '\': ' + str(e))