diff --git a/hshetl/__init__.py b/hshetl/__init__.py index 1da01490c1b2eb523d489bd96504fd832de39871..952627efac3b3395c2510da75b23e6307f51cf0d 100644 --- a/hshetl/__init__.py +++ b/hshetl/__init__.py @@ -166,20 +166,22 @@ class NameResolver(object): '''Resolve the object from the repository.''' if isinstance(self.argument_value, str) or isinstance(self.argument_value, unicode): obj = self.repository.get(self.argument_value) + if obj is None: + raise ConfigurationException('No reference with name {} known to the repository {}'.format(self.argument_value, self.repository.__class__.__name__)) elif self.argument_value == None: return None # If None is given in kwargs - we assume, that this is intended to be possible. elif not isinstance(self.argument_value, self.repository.cls): raise ConfigurationException('The argument {} must be an instance of {} or a string/unicode that identifies it. Encountered {}'.format(self.argument_name, self.repository.cls.__name__, repr(self.argument_value))) else: obj = self.argument_value - if self.add: + if self.add and obj is not None: self.repository.add(obj) - elif not self.repository.has(obj): + elif not obj in self.repository.itervalues(): raise ConfigurationException('The object is not known by its repository and the resolver should not add it to the repository. Thats an error, because you gave us a object of the correct type, that is unknown.') self.argument_value = None return obj -class AbstractRepository(OrderedDict, object): +class AbstractRepository(object): '''An base class for repository classes. May act as singleton inside hshetl. @@ -189,7 +191,7 @@ class AbstractRepository(OrderedDict, object): def __init__(self, items = []): '''Default constructor.''' '''A dictionary that is internally used to store entity schemes.''' - super(AbstractRepository, self).__init__() + self._container = OrderedDict() self.extend(items) if not hasattr(self, 'cls') or not hasattr(self, 'ref_attr'): raise Exception('Repository unable to work, class or ref_attr not known.') @@ -200,19 +202,6 @@ class AbstractRepository(OrderedDict, object): for obj in list(self.itervalues()): self.remove(obj) - def get(self, ref): - '''Return the entity with the assigned name.''' - try: - return self[ref] - except KeyError: - raise UnknownNameReferenceException('The {} doesn\'t store the object with reference {}'.format(self.__class__.__name__, ref)) - - def has(self, ref): - '''Return if the given reference or object is known by the repository''' - if isinstance(ref, self.cls): - ref = getattr(ref, self.ref_attr) - return ref in self - def extend(self, items): '''Extend the repository with a list of entities''' for item in items: @@ -227,20 +216,38 @@ class AbstractRepository(OrderedDict, object): ''' if not isinstance(obj, self.cls): raise TypeError('The Manager only accepts instances that inherit from {}'.format(self.cls.__name__)) - if self.has(obj): return - if self.has_key(getattr(obj, self.ref_attr)) and self.get(getattr(obj, self.ref_attr)) != obj: + if getattr(obj, self.ref_attr) in self._container: return + if getattr(obj, self.ref_attr) in self._container and self.get(getattr(obj, self.ref_attr)) != obj: raise DuplicateNameException('The manager already stores a entity named ' + getattr(obj, self.ref_attr)) logging.debug('Adding object with name {} into the {}'.format(getattr(obj, self.ref_attr), self.__class__.__name__)) - self[getattr(obj, self.ref_attr)] = obj + self._container[getattr(obj, self.ref_attr)] = obj def remove(self, obj): '''Remove a object from the repository and close it before.''' if isinstance(obj, str): obj = self.get(obj) - del self[getattr(obj, self.ref_attr)] + del self._container[getattr(obj, self.ref_attr)] + + def itervalues(self): + return self._container.itervalues() + + def iteritems(self): + return self._container.iteritems() + + def iterkeys(self): + return self._container.iterkeys() - def __repr__(self): - return 'AbtractRepository: ' + super(AbstractRepository, self).__repr__() + def __iter__(self): + return self._container.__iter__() + + def __contains__(self, item): + return item in self._container + + def __len__(self): + return self._container.__len__() + + def get(self, ref): + return self._container.get(ref, None) import entities diff --git a/hshetl/entities.py b/hshetl/entities.py index c09718451dee6043f53e29bf911bab4227d4d646..dd1023b33bc30be80343d1f3933b38d4673d5d91 100644 --- a/hshetl/entities.py +++ b/hshetl/entities.py @@ -13,6 +13,7 @@ import hashlib import logging from uuid import uuid4 as uuid from exc import UnknownPropertyException, DuplicatedJoinIdException, DuplicatedSystemIdException, DataTypeConversionException, MappingException, ContainerPropertiesException +from hshetl.exc import ConfigurationException __all__ = ['COLLISION_HANDLING_BREAKALL', 'COLLISION_HANDLING_BREAKALL', 'COLLISION_HANDLING_BREAKCONTAINER', 'COLLISION_HANDLING_BREAKNEVER', 'OPERATION_INSERT', 'OPERATION_UPDATE', 'OPERATION_DELETE', 'OPERATION_UNALTERED', 'OPERATION_UNKNOWN', @@ -72,6 +73,7 @@ class Entity(AbstractRepository): def load(self, data, container): '''Merge the assigned target and load dictionaries into entities.''' container = NameResolver(self, 'container', container, add = False)() + if container.name not in self: raise ConfigurationException('Can not load into containers that are not known by the entity.') logging.info('Load {} records into the entity {}.'.format(len(data), self.name)) for raw_record in data: self._load_record(raw_record, container) @@ -88,7 +90,7 @@ class Entity(AbstractRepository): container.container_collide(Record(join_identifier, self.properties, self.itervalues())) return - if self.record_repository.has(join_identifier): + if join_identifier in self.record_repository: record = self.record_repository.get(join_identifier) else: record = Record(join_identifier, self.properties, self.itervalues()) diff --git a/hshetl/test/unit/test_entities.py b/hshetl/test/unit/test_entities.py index 8616235c7c8c1ae17b3f032bc20d3638c9afdaeb..865cb71b65924786d06de53cc337fae1965fb6d0 100644 --- a/hshetl/test/unit/test_entities.py +++ b/hshetl/test/unit/test_entities.py @@ -18,7 +18,7 @@ class EntityTest(unittest.TestCase): def test_load(self): self.entity.add(self.container) - self.assertRaises(UnknownNameReferenceException, self.entity.load, [], 'Unknown') + self.assertRaises(ConfigurationException, self.entity.load, [], 'Unknown') unregisteredContainer = Container({}) self.assertRaises(ConfigurationException, self.entity.load, [], unregisteredContainer) self.entity._load_record = Mock() diff --git a/hshetl/test/unit/test_jobs.py b/hshetl/test/unit/test_jobs.py index bed47a6871ea5bcec77d1b51a0c40e5e343e831e..50b7409ef968a58d2d6c6f433ae4756563f796f2 100644 --- a/hshetl/test/unit/test_jobs.py +++ b/hshetl/test/unit/test_jobs.py @@ -55,7 +55,7 @@ class TestJobs(unittest.TestCase): self.assertEqual(job.connector, self.connector_mock) self.assertRaises(ConfigurationException, ConnectorJob, connector = []) self.assertRaises(ConfigurationException, ConnectorJob, connector = dict()) - self.assertRaises(UnknownNameReferenceException, ConnectorJob, connector = 'UnknownConnector') + self.assertRaises(ConfigurationException, ConnectorJob, connector = 'UnknownConnector') conn = Mock(spec = connectors.FileConnector) conn.name = 'InlineDefined' ConnectorJob(connector = conn) @@ -68,7 +68,7 @@ class TestJobs(unittest.TestCase): self.assertEqual(job.entity, self.entity_mock) self.assertRaises(ConfigurationException, EntityJob, entity = []) self.assertRaises(ConfigurationException, EntityJob, entity = dict()) - self.assertRaises(UnknownNameReferenceException, EntityJob, entity = 'UnknownEntity') + self.assertRaises(ConfigurationException, EntityJob, entity = 'UnknownEntity') ent = Mock(spec = entities.Entity) ent.name = 'InlineDefined' EntityJob(entity = ent)