From 89752c6c69d965b644d0d8b46151315071443c31 Mon Sep 17 00:00:00 2001 From: Dennis Ahrens <dennis.ahrens@hs-hannover.de> Date: Fri, 14 Mar 2014 17:14:10 +0100 Subject: [PATCH] [BUGFIX] SQL loader works correctly. Insert added empty record if there was no data supplied and update and delete had a bug related to the constraits creation. --- hshetl/loaders.py | 6 ++++-- hshetl/test/functional/test_sync_file2sqlite.py | 9 ++++++--- hshetl/test/unit/test_loaders.py | 4 ++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/hshetl/loaders.py b/hshetl/loaders.py index 3e8cddf..3b21faf 100644 --- a/hshetl/loaders.py +++ b/hshetl/loaders.py @@ -217,6 +217,7 @@ class SqlAlchemyLoader(AbstractLoader): def _insert(self, connection, data): '''Creates a sql insert statement using sqlalchemy and executes it on the connection.''' + if len(data) == 0: return values = [] for record in data: values.append(record.to_dict(remove_system_identifiers = False)) @@ -228,7 +229,7 @@ class SqlAlchemyLoader(AbstractLoader): '''Creates a sql update statement using sqlalchemy and executes it on the connection.''' for record in data: statement = self.table.update(values = record.to_dict()) - self._add_where_constraints(statement, record) + statement = self._add_where_constraints(statement, record) logging.debug('executing sql update query: ' + str(statement)) connection.execute(statement) @@ -237,7 +238,7 @@ class SqlAlchemyLoader(AbstractLoader): '''Creates a sql delete statement using sqlalchemy and executes it on the connection.''' for record in data: statement = self.table.delete() - self._add_where_constraints(statement, record) + statement = self._add_where_constraints(statement, record) logging.debug('executing sql delete query: ' + str(statement)) connection.execute(statement) @@ -256,6 +257,7 @@ class SqlAlchemyLoader(AbstractLoader): statement = statement.where(self.table.c.get(column) == value) else: raise ConfigurationException('Value for identifier \'' + str(column) + '\' empty.') + return statement @yamlify diff --git a/hshetl/test/functional/test_sync_file2sqlite.py b/hshetl/test/functional/test_sync_file2sqlite.py index 40cba1a..6efaa16 100644 --- a/hshetl/test/functional/test_sync_file2sqlite.py +++ b/hshetl/test/functional/test_sync_file2sqlite.py @@ -11,9 +11,10 @@ class TestSyncFileToSQLiteSync(TestBaseClass): super(TestSyncFileToSQLiteSync, self).setUp() self.conn = sqlite3.connect(self.test_working_directory + 'person.sqlite3') self.cursor = self.conn.cursor() + self.cursor.execute('DROP TABLE IF EXISTS user;') self.cursor.execute('CREATE TABLE user (id INTEGER PRIMARY KEY, name VARCHAR(50), inactive BOOL, foreign_id INT, telephone INT);') self.conn.commit() - sys.argv = ['hshetl', '-d', self.test_working_directory] + sys.argv = ['hshetl', '-v', '-d', self.test_working_directory] def tearDown(self): super(TestSyncFileToSQLiteSync, self).tearDown() @@ -38,13 +39,15 @@ class TestSyncFileToSQLiteSync(TestBaseClass): def test_sync_updates_outdated_data_set(self): csv_file = open(self.test_working_directory + 'data/person.csv', 'w') - csv_file.write('p_num,name,retired,telephone\n100,Hanna Ferguson,0,12345678') + csv_file.write('p_num,name,retired,telephone\n100,Hanna Ferguson,0,12345678\n101,Willy Williams,0,87654321') csv_file.close() self.cursor.execute('INSERT INTO user (foreign_id,inactive,name,telephone,id) VALUES(100, 1, \'Hanae Ferguson\', 67466035,20)') + self.cursor.execute('INSERT INTO user (foreign_id,inactive,name,telephone,id) VALUES(101, 1, \'Willi Williams\', 67466035,21)') self.conn.commit() Controller().configure().execute() self.cursor.execute('SELECT * FROM user') self.assertEqual(self.cursor.fetchone(), (20, u'Hanna Ferguson', 0, 100, 12345678)) + self.assertEqual(self.cursor.fetchone(), (21, u'Willy Williams', 0, 101, 87654321)) def test_sync_deletes_outdated_data_set(self): csv_file = open(self.test_working_directory + 'data/person.csv', 'w') @@ -54,7 +57,7 @@ class TestSyncFileToSQLiteSync(TestBaseClass): self.conn.commit() Controller().configure().execute() self.cursor.execute('SELECT * FROM user') - self.assertEqual(self.cursor.fetchone(), None, 'A data set that does not exist in the source system was not deleted.') + self.assertEqual(self.cursor.fetchone(), None) def test_sync_handles_insert_update_and_delete_simultaneously(self): expected_result = [(1, u'Hanae Ferguson', 1, 100, 67466035), diff --git a/hshetl/test/unit/test_loaders.py b/hshetl/test/unit/test_loaders.py index ddfc479..1ef30af 100644 --- a/hshetl/test/unit/test_loaders.py +++ b/hshetl/test/unit/test_loaders.py @@ -131,7 +131,7 @@ class TestSqlAlchemyLoader(unittest.TestCase): Mock(spec = sqlalchemy.sql.expression.Update), Mock(spec = sqlalchemy.sql.expression.Update)] self.table_mock.update = Mock(side_effect = table_update_side_effect) - self.etl_loader._add_where_constraints = Mock() + self.etl_loader._add_where_constraints = Mock(side_effect = table_update_side_effect) fixture = copy.deepcopy(test.fixtures['loader_test_loading']) del fixture[0]['Name'] del fixture[1]['Name'] @@ -161,7 +161,7 @@ class TestSqlAlchemyLoader(unittest.TestCase): Mock(spec = sqlalchemy.sql.expression.Delete), Mock(spec = sqlalchemy.sql.expression.Delete)] self.table_mock.delete = Mock(side_effect = table_delete_side_effect) - self.etl_loader._add_where_constraints = Mock() + self.etl_loader._add_where_constraints = Mock(side_effect = table_delete_side_effect) fixture = copy.deepcopy(test.fixtures['loader_test_loading']) self.result.delete[0].get_container_identifier = Mock(return_value = fixture[0]) self.result.delete[1].get_container_identifier = Mock(return_value = fixture[1]) -- GitLab