From e1f4a27cce75b20a37ad24610e58a08642c7e5dd Mon Sep 17 00:00:00 2001 From: Maximilian Schulz <maximilian.schulz@hs-hannover.de> Date: Fri, 28 Mar 2014 10:26:54 +0100 Subject: [PATCH] [TASK] Improves commenting for sphinx Comments now contain a method signature and parameters with types for all constructors available. Also includes a small fix to the editor.py --- hshetl/connectors.py | 62 ++++++++++-- hshetl/editor.py | 11 +-- hshetl/entities.py | 54 ++++++++++- hshetl/extractors.py | 90 +++++++++-------- hshetl/jobs.py | 216 ++++++++++++++++++++++------------------- hshetl/loaders.py | 53 +++++----- hshetl/transformers.py | 53 ++++++---- 7 files changed, 336 insertions(+), 203 deletions(-) diff --git a/hshetl/connectors.py b/hshetl/connectors.py index e582c05..d563a71 100644 --- a/hshetl/connectors.py +++ b/hshetl/connectors.py @@ -57,6 +57,17 @@ __all__ = ['AbstractConnector', 'FileConnector', 'LdapConnector', 'OracleConnect class AbstractConnector(object): '''Connectors provide an api for extractors and loaders to access databases. + .. function:: __init__(uri[, name[,encoding]]) + + Constructor + + :param uri: Host or path + :param name: Name of the connector + :param encoding: encoding to be used, standard is utf-8 + :type uri: string + :type name: string or None + :type encoding: string + The connector is a context manager which means you can and should use it in with statements. This way the connection will close any connection handles automatically. @@ -67,7 +78,6 @@ class AbstractConnector(object): with connector as connection: do_something_with_real_handle(connection) do_something_with_connetor(connector) - ''' def __init__(self, uri, name = None, encoding = 'utf-8'): @@ -118,7 +128,13 @@ class AbstractConnector(object): class FileConnector(AbstractConnector): '''A connector to a data source of type file. - ``**kwargs`` accepts arguments from :class:`.AbstractConnector`. + .. function:: __init__([mode[,**kwargs]]) + + Constructor + + :param mode: mode for file (r or w, where standard is r) + :param **kwargs: accepts arguments from :class:`.AbstractConnector` + :type mode: string ''' @@ -156,7 +172,21 @@ class FileConnector(AbstractConnector): class LdapConnector(AbstractConnector): '''A connector to a data source of type ldap server. - ``**kwargs`` accepts arguments from :class:`.AbstractConnector`. + .. function:: __init__(uri[, base = None[, scope = 'subtree'[, page_size = False[, start_tls = True[, **kwargs]]]]]) + + Constructor + + :param uri: host url + :param base: LDAP-base + :param scope: LDAP-scope + :param page_size: If set the extraction executes a paged query (for MS AD) + :param start_tls: use LDAP-TLS + :param **kwargs: arguments from :class:`.AbstractConnector` + :type uri: string + :type base: string or None + :type scope: string + :type page_size: boolean + :type start_tls: boolean ''' @@ -221,7 +251,13 @@ class LdapConnector(AbstractConnector): class SqlAlchemyConnector(AbstractConnector): '''Encapsulate the connection to SQL databases. - ``**kwargs`` accepts arguments from :class:`.AbstractConnector`. + .. function:: __init__(uri[, **kwargs]) + + Constructor + + :param uri: host or path + :param **kwargs: arguments from :class:`.AbstractConnector` + :type uri: string ''' @@ -279,9 +315,13 @@ class SqlQueryLanguageSupportMixin(object): @yamlify class OracleConnector(SqlAlchemyConnector, SqlQueryLanguageSupportMixin): - '''A connector to a oracle database. + '''A connector to an oracle database. + + .. function:: __init__([**kwargs]) - ``**kwargs`` accepts arguments from :class:`.AbstractConnector` and :class:`SqlAlchemyConnector`. + Constructor + + :param **kwargs: arguments from :class:`.AbstractConnector` and :class:`SqlAlchemyConnector` ''' @@ -313,7 +353,15 @@ class OracleConnector(SqlAlchemyConnector, SqlQueryLanguageSupportMixin): class MySQLConnector(SqlAlchemyConnector): '''A connector to a mysql database. - ``**kwargs`` accepts arguments from :class:`.AbstractConnector` and :class:`SqlAlchemyConnector`. + .. function:: __init__(uri, encoding[, **kwargs]) + + Constructor + + :param uri: host + :param encoding: encoding to be used + :param **kwargs: arguments from :class:`.AbstractConnector` and :class:`SqlAlchemyConnector` + :type uri: string + :type encoding: string ''' diff --git a/hshetl/editor.py b/hshetl/editor.py index ec93a1b..f4d1808 100644 --- a/hshetl/editor.py +++ b/hshetl/editor.py @@ -23,6 +23,7 @@ def json_encode_class(cls): "form": {"display": "textfield"}} if prop in required: cfg["properties"][prop]["defaults"] = "foo" + return cfg def dump_schema(): yaml_subclasses = {} @@ -34,22 +35,20 @@ def dump_schema(): "entities": ["!entity"], "jobs": []}} classes = [] - for cls in sorted(yaml_subclasses): - print(yaml_subclasses[cls]) - continue - return - if True: + for cls in yaml_subclasses.itervalues(): if "job" in cls.__name__.lower(): docs["children"]["jobs"].append(cls.yaml_tag) elif "connector" in cls.__name__.lower(): docs["children"]["connectors"].append(cls.yaml_tag) - classes.append(json_encode_class(cls)) + if cls.yaml_tag != "!void": + classes.append(json_encode_class(cls)) with open('editor_data.js','w') as f: print("\"use strict\";\n\nvar hshetl_editor_data = " + json.dumps({"documents": docs, "classes": classes}, indent=4, separators=(',', ': ')), file=f) + print("editor_data.js successfully generated!") if __name__ == '__main__': diff --git a/hshetl/entities.py b/hshetl/entities.py index d068a41..fdc9587 100644 --- a/hshetl/entities.py +++ b/hshetl/entities.py @@ -35,6 +35,19 @@ OPERATION_UNKNOWN = 'UNKNOWN' class Entity(AbstractRepository): '''Entity contain all information about an entity. + .. function:: __init__(properties[, name[, join_properties[, containers]]]) + + Constructor + + :param properties: properties for this entity + :param name: name of the entity, standard is a generated unicode uuid + :param join_properties: list of join properties + :param containers: list of containers + :type properties: dict + :type name: string + :type join_properties: list + :type containers: list + The entity stores information about the properties, their data types and names in the different containers. It can also convert raw values to the desired data type and @@ -129,7 +142,23 @@ class RecordRepository(AbstractRepository): class Container(list): - '''A container is the source or the target of data related to an entity.''' + '''A container is the source or the target of data related to an entity. + + .. function:: __init__(properties[, name[, identifiers[, mapping[, collision_handling]]]]) + + Constructor + + :param properties: propertydictionary + :param name: name of container, standard is a generated unicode uuid + :param identifiers: list of identifiers + :param mapping: mapping dictionary + :param collision_handling: collision handling (breakall, breakjoin, breakcontainer or breaknever) + :type properties: dict + :type name: string or None + :type identifiers: list + :type mapping: dict + :type collision_handling: string + ''' def __init__(self, properties, name = None, identifiers = [], mapping = {}, collision_handling = COLLISION_HANDLING_BREAKALL): self.name = (name if name is not None else unicode(uuid())) @@ -297,7 +326,19 @@ class PropertyConverter(object): class Record(object): - '''Records merge and store data from extractions''' + '''Records merge and store data from extractions + + .. function:: __init__(identifier, properties[, containers]) + + Constructor + + :param identifier: hash value of all join fields + :param properties: propertydictionary + :param containers: list of containers + :type identifier: string + :type properties: dict + :type containers: list + ''' def __init__(self, identifier, properties, containers = []): '''Sets initial values for the entity. @@ -455,6 +496,15 @@ entity_repository = EntityRepository() class Result(object): '''This class represents a Result. + .. function:: __init__(entity, source) + + Constructor + + :param entity: the entity to be representated + :param source: system to fetch data from + :type entity: :class:`Entity` + :type source: string + A Result describes what records between the source and the target has to be updated, inserted or deleted. diff --git a/hshetl/extractors.py b/hshetl/extractors.py index 0fe24e0..4afe88c 100644 --- a/hshetl/extractors.py +++ b/hshetl/extractors.py @@ -39,10 +39,12 @@ __all__ = ['AbstractExtractor', 'SqlAlchemyExtractor', 'CsvExtractor', 'LdapExtr class AbstractExtractor(object): '''A base class for extractors. - Construction: + .. function:: __init__([connector]) - :param hshetl.connectors.AbstractConnector connector: - The connector this extractor uses to fetch the data. + Constructor + + :param connector: connector instance + :type connector: :class:`.AbstractExtractor` ''' @@ -87,12 +89,13 @@ class AbstractExtractor(object): class SqlAlchemyExtractor(AbstractExtractor): '''An extractor for SqlAlchemyConnectors. - Construction: + .. function:: __init__(query[, *args[, **kwargs]]) + + Constructor - :param str query: - The query that fetches the data from the database. - :param dict ``**kwargs``: - Accepts parameters from :class:`.AbstractExtractor`. + :param query: query that fetches data from the database + :param **kwargs: accepts parameters from :class:`.AbstractExtractor`. + :type query: string YAML definition sample: @@ -136,20 +139,22 @@ class SqlAlchemyExtractor(AbstractExtractor): class LdapExtractor(AbstractExtractor): '''An extractor for LdapConnectors. - Construction: - - :param str base: - The base to start the extraction. If None the base defined in the connector is used. - :param str scope: - The ldap scope for the extraction. Possibile values: ``base``, ``onelevel`` and ``subtree``. If None the scope defined in the connector is used. - :param str ldap_filter: - The ldap filter for the extraction. Default: ``(objectclass=*)`` - :param list attributes: - The attributes fetched in this extraction. Default: ``[]`` - :param int page_size: - If set the extraction executes a paged query as defined in RFC 2696. The value set is the page size used. Use this when you query a MS AD. - :param dict ``**kwargs``: - Accepts parameters from :class:`.AbstractExtractor`. + .. function:: __init__([base[, scope[, ldap_filter[, attributes[, page_size[, *args[, **kwargs]]]]]]]) + + Constructor + + :param base: The base to start the extraction. If None the base defined in the connector is used. + :param scope: The ldap scope for the extraction. Possibile values: ``base``, ``onelevel`` and ``subtree``. If None the scope defined in the connector is used. + :param ldap_filter: The ldap filter for the extraction. Default: ``(objectclass=*)`` + :param attributes: The attributes fetched in this extraction. Default: ``[]`` + :param page_size: If set the extraction executes a paged query as defined in RFC 2696. The value set is the page size used. Use this when you query a MS AD. + :param **kwargs: Accepts parameters from :class:`.AbstractExtractor`. + + :type base: string or None + :type scope: string or None + :type ldap_filter: string or None + :type attributes: list + :type page_size: int or None YAML definition sample: @@ -270,12 +275,13 @@ class CsvExtractor(AbstractExtractor): But also other sources might be interesting. E.g. HTTP or SSH. - Construction: + .. function:: __init__([dialect[, *args[, **kwargs]]]) - :param hshetl.Dialect dialect: - The CSV dialect that ill be used for CSV style. - :param dict ``**kwargs``: - Accepts parameters from :class:`.AbstractExtractor`. + Constructor + + :param dialect: CSV dialect to be use for CSV style + :param **kwargs: Accepts parameters from :class:`.AbstractExtractor`. + :type dialect: :class:`hshetl.Dialect` YAML definition sample: @@ -340,20 +346,22 @@ class FakerExtractor(AbstractExtractor): Have a look at `faker <https://github.com/joke2k/faker>`_ for more information about the possibilities here. - Construction: - - :param int number: - The number of records faked by extractor. Default is 100. - :param dict properties: - The properties that will be created. As value you can directly hand over the methods available in faker. They will be executed using eval(). - :param list providers: - The providers that will be available. - :param str locale: - The locale handed over to the faker factory. - :param str seed: - Use a seed to get the same data each time. - :param dict ``**kwargs``: - Accepts parameters from :class:`.AbstractExtractor` but note that the connector will never be used. + .. function:: __init__([number[, properties[, providers[, locale[, seed[, *args[, **kwargs]]]]]]]) + + Constructor + + :param number: The number of records faked by extractor. Default is 100. + :param properties: The properties that will be created. As value you can directly hand over the methods available in faker. They will be executed using eval(). + :param providers: The providers that will be available. + :param locale: The locale handed over to the faker factory. + :param seed: Use a seed to get the same data each time. + :param **kwargs: Accepts parameters from :class:`.AbstractExtractor` but note that the connector will never be used. + + :type number: int + :type properties: dict or None + :type providers: list or None + :type locale: string or None + :type seed: string or None YAML definition sample: diff --git a/hshetl/jobs.py b/hshetl/jobs.py index 4e3c0ba..e69ef24 100644 --- a/hshetl/jobs.py +++ b/hshetl/jobs.py @@ -56,14 +56,16 @@ class Job(object): If you do not find a job matching your requirements extend this class. - Construction: + .. function:: __init__([name[, description[, interactive]]]) - :param unicode name: - The name of the job. - :param unicode description: - The description of the job. - :param bool interactive: - If ``True`` you will be asked before the execution. When launching from cli with option ``--interactive`` this value will always be ``True``. + Constructor + + :param name: The name of the job. + :param description: The description of the job. + :param interactive: If ``True`` you will be asked before the execution. When launching from cli with option ``--interactive`` this value will always be ``True``. + :type name: unicode + :type description: unicode + :type interactive: bool YAML definition sample: @@ -148,12 +150,13 @@ class JobList(Job, list): Executing a JobList will execute all children recursively. - Construction: + .. function:: __init__([jobs[, *args[, **kwargs]]]) + + Constructor - :param list jobs: - The jobs for the list. - :param dict ``**kwargs``: - Accepts parameters from :class:`.Job` and :class:`list`. + :param jobs: The jobs for the list. + :param **kwargs: Accepts parameters from :class:`.Job` and :class:`list`. + :type jobs: list YAML definition sample: @@ -188,12 +191,13 @@ class ConnectorJob(Job): repository, or a connector object which will be added to the connector repository after creation. - Construction: + .. function:: __init__([connector[, *args[, **kwargs]]]) + + Constructor - :param hshetl.connector.AbstractConnector connector: - The connector used by the job. Can be None - depends on the inheriting class if required or not. - :param dict ``**kwargs``: - Accepts parameters from :class:`.Job`. + :param connector: The connector used by the job. Can be None - depends on the inheriting class if required or not. + :param **kwargs: Accepts parameters from :class:`.Job`. + :type connector: :class:`hshetl.connector.AbstractConnector` ''' @@ -210,12 +214,13 @@ class ConnectorJob(Job): class BashJob(Job): '''A job for executing stuff using the unix shell. - Construction: + .. function:: __init__(command[, *args[, **kwargs]]) - :param str command: - The command that will be executed. - :param dict ``**kwargs``: - Accepts parameters from :class:`.Job`. + Constructor + + :param command: The command that will be executed. + :param **kwargs: Accepts parameters from :class:`.Job`. + :type command: string YAML definition sample: @@ -249,12 +254,13 @@ class BashJob(Job): class QueryJob(ConnectorJob): '''This job resolves a sql query you will not use it directly. - Construction: + .. function:: __init__(sql[, *args[, **kwargs]]) + + Constructor - :param str sql: - The query that will be execute or a reference to a file, if it starts with ``file://``. - :param dict ``**kwargs``: - Accepts parameters from :class:`.ConnectorJob` and :class:`.Job`. + :param: sql: The query that will be execute or a reference to a file, if it starts with ``file://``. + :param **kwargs: Accepts parameters from :class:`.ConnectorJob` and :class:`.Job`. + :type sql: string TODO: All SQL is stored in RAM - what about big files? @@ -281,10 +287,13 @@ class QueryJob(ConnectorJob): class SqlQueryJob(QueryJob): '''Executes a SQL query using a connector. - Construction: + .. function:: __init__(connector[, *args[, **kwargs]]) + + Constructor - :param dict ``**kwargs``: - Accepts parameters from :class:`.QueryJob`, :class:`.ConnectorJob` and :class:`.Job`. + :param connector: Connector for sql sources + :param **kwargs: Accepts parameters from :class:`.QueryJob`, :class:`.ConnectorJob` and :class:`.Job`. + :type connector: :class:`hshetl.connectors.SqlAlchemyConnector` YAML definition sample: @@ -334,6 +343,14 @@ class SqlQueryJob(QueryJob): class PlSqlQueryJob(QueryJob): '''Executes a PL/SQL query on the given connector. + .. function:: __init__(connector[, *args[, **kwargs]]) + + Constructor + + :param connector: Connector for sql sources + :param **kwargs: Accepts parameters from :class:`.QueryJob`, :class:`.ConnectorJob` and :class:`.Job`. + :type connector: :class:`hshetl.connectors.SqlQueryLanguageSupportMixin` + Works in the same way like :class:`SqlQueryJob`, but can handle PL/SQL and PLPG/SQL on oracle and postrgresql databases. @@ -360,11 +377,11 @@ class PlSqlQueryJob(QueryJob): class EntityJob(Job): '''A job regarding entity based operations. + .. function:: __init__(entity[, *args[, **kwargs]]) - Construction: - - :param hshetl.entities.Entity entity: - The entity that that job will load data for. + :param entity: The entity that that job will load data for. + :param **kwargs: Accepts parameters from :class:`Job` + :type entity: :class:`hshetl.entities.Entity` or string ''' @@ -382,21 +399,22 @@ class EntityJob(Job): class ExtractionJob(EntityJob, ConnectorJob): '''A job that extracts data from a source and loads it into the related entities record repository. - Construction: - - :param hshetl.extractors.AbstractExtractor extractor: - The extractor that will fetch the data. - :param unicode name: - The name of the job. This name can be used as referenced for transformers and loaders. - :param list identifiers: - The primary key columns of your data. An empty list assumes that all fields are used as key. - :param dict mapping: - The mapping from this source to entity definition. ``{'name_in_data_source': 'name_in_entity'}`` - :param str collision_handling: - How to handle collisions when loading the data into the entity. Possible values: ``BREAKALL``, ``BREAKCONTAINER``, - ``BREAKJOIN`` and ``BREAKNEVER`` - :param dict ``**kwargs``: - Accepts parameters from :class:`.EntityJob`, :class:`.ConnectorJob` and :class:`.Job`. + .. function:: __init__(extractor[, name[, identifiers[, mapping[, collision_handling[, *args[, **kwargs]]]]]]) + + Constructor + + :param extractor: The extractor that will fetch the data. + :param name: The name of the job. This name can be used as referenced for transformers and loaders. + :param identifiers: The primary key columns of your data. An empty list assumes that all fields are used as key. + :param mapping: The mapping from this source to entity definition. ``{'name_in_data_source': 'name_in_entity'}`` + :param collision_handling: How to handle collisions when loading the data into the entity. Possible values: ``BREAKALL``, ``BREAKCONTAINER``, ``BREAKJOIN`` and ``BREAKNEVER`` + :param **kwargs: Accepts parameters from :class:`.EntityJob`, :class:`.ConnectorJob` and :class:`.Job`. + + :type extractor: :class:`hshetl.extractors.AbstractExtractor` + :type name: unicode + :type identifiers: list + :type mapping: dict + :type collision_handling: string Note: The connector can be passed to the job, or directly to the extractor. If it is missing the job will fail on execution and not on construction! @@ -457,20 +475,20 @@ class ExtractionJob(EntityJob, ConnectorJob): class TransformerJob(EntityJob, ConnectorJob): '''A job that does something with entities in a repository. - Construction: - - :param hshetl.transformers.AbstractTransformer transformer: - The transformer that will do something with the data. - :param unicode name: - The name of the job. This name can be used as referenced for following transformers and loaders. - :param unicode source: - The reference to the source data container of the entity - normally defined with the name of a - :class:`.ExtractionJob` or :class:`.TransformerJob`. - :param unicode target: - The reference to the target data container of the entity - normally defined with the name of a - :class:`.ExtractionJob` or :class:`.TransformerJob`. - :param dict ``**kwargs``: - Accepts parameters from :class:`.EntityJob`, :class:`.ConnectorJob` and :class:`.Job`. + .. function:: __init__(transformer[, entity[, source[, target[, *args[, **kwargs]]]]]) + + Constructor + + :param transformer: The transformer that will do something with the data. + :param name: The name of the job. This name can be used as referenced for following transformers and loaders. + :param source: The reference to the source data container of the entity - normally defined with the name of a :class:`.ExtractionJob` or :class:`.TransformerJob`. + :param target: The reference to the target data container of the entity - normally defined with the name of a :class:`.ExtractionJob` or :class:`.TransformerJob`. + :param **kwargs: Accepts parameters from :class:`.EntityJob`, :class:`.ConnectorJob` and :class:`.Job`. + + :type transformer: :class:`hshetl.transformers.AbstractTransformer` + :type name: unicode + :type source: unicode + :type target: unicode YAML definition sample: @@ -519,13 +537,13 @@ class TransformerJob(EntityJob, ConnectorJob): class InspectJob(EntityJob): '''Inspection of result in a interactive python shell. - Construction: + .. function:: __init__(source[, *args[, **kwargs]]) + + Constructor - :param unicode source: - The reference to the source data container of the entity - normally defined with the name of a - :class:`.ExtractionJob` or :class:`.TransformerJob`. - :param dict ``**kwargs``: - Accepts parameters from :class:`.EntityJob` and :class:`.Job`. + :param source: The reference to the source data container of the entity - normally defined with the name of a :class:`.ExtractionJob` or :class:`.TransformerJob`. + :param **kwargs: Accepts parameters from :class:`.EntityJob` and :class:`.Job`. + :type source: unicode YAML definition sample: @@ -593,17 +611,15 @@ class InspectJob(EntityJob): class LoadJob(EntityJob, ConnectorJob): '''A job that loads data into a data source. - Construction: + .. function:: __init__(loader, source[, *args[, **kwargs]]) - :param hshetl.loaders.AbstractLoader loader: - The loader that will write the data. - :param unicode name: - The name of the job. This name can be used as referenced for transformers and loaders. - :param unicode source: - The reference to the data container of the entity - normally defined with the name of a - :class:`.ExtractionJob` or :class:`.TransformerJob`. - :param dict ``**kwargs``: - Accepts parameters from :class:`.EntityJob`, :class:`.ConnectorJob` and :class:`.Job`. + Constructor + + :param loader: The loader that will write the data. + :param source: The reference to the data container of the entity - normally defined with the name of a :class:`.ExtractionJob` or :class:`.TransformerJob`. + :param **kwargs: Accepts parameters from :class:`.EntityJob`, :class:`.ConnectorJob` and :class:`.Job`. + :type loader: :class:`hshetl.loaders.AbstractLoader` + :type source: unicode Note: The connector can be passed to the job, or directly to the loader. If it is missing the job will fail on execution and not on construction! @@ -671,18 +687,18 @@ class SynchronizationJob(JobList, EntityJob): 3. :class:`.TransformerJob` that will execute a :class:`hshetl.transformer.CompareTransformer` for source and target 4. :class:`.LoadJob` that will write the data into the target system. - Construction: + .. function:: __init__(source, target[, inspect[, **kwargs]]) + + Constructor + + :param source: This dict will be used as is to construct the :class:`.ExtractionJob` for the source system. + :param target: This dict will be used to construct the :class:`.ExtractionJob` for the target system. It also has to contain a key called loader, that will be removed and used for the construction of the :class:`.LoadJob`. + :param inspect: If ``True`` an InspectJob is added after the :class:`.TransformerJob` + :param **kwargs: Accepts parameters from :class:`.EntityJob` and :class:`.JobList`. - :param dict source: - This dict will be used as is to construct the :class:`.ExtractionJob` for the source system. - :param dict target: - This dict will be used to construct the :class:`.ExtractionJob` for the target system. It also - has to contain a key called loader, that will be removed and used for the construction of the - :class:`.LoadJob`. - :param bool inspect: - If ``True`` an InspectJob is added after the :class:`.TransformerJob` - :param dict ``**kwargs``: - Accepts parameters from :class:`.EntityJob` and :class:`.JobList`. + :type source: dict + :type target: dict + :type inspect: bool YAML sample definition: @@ -750,18 +766,14 @@ class CopyJob(JobList, EntityJob): Note that this job will never apply any mapping to the target data source. Just define your entity identical to the target data source. - Construction: - - :param dict source: - This dict will be used as is to construct the :class:`.ExtractionJob` for the source data source. - :param dict target: - This dict will be used to construct the :class:`.ExtractionJob` for the target data source. It also - has to contain a key called loader, that will be removed and used for the construction of the - :class:`.LoadJob`. - :param bool inspect: - If ``True`` an InspectJob is added after the :class:`.TransformerJob` - :param dict ``**kwargs``: - Accepts parameters from :class:`.EntityJob` and :class:`.JobList`. + .. function:: __init__(source, target[, inspect[, **kwargs]]) + + Constructor + + :param source: This dict will be used as is to construct the :class:`.ExtractionJob` for the source data source. + :param target: This dict will be used to construct the :class:`.ExtractionJob` for the target data source. It also has to contain a key called loader, that will be removed and used for the construction of the :class:`.LoadJob`. + :param inspect: If ``True`` an InspectJob is added after the :class:`.TransformerJob` + :param **kwargs: Accepts parameters from :class:`.EntityJob` and :class:`.JobList`. YAML sample definition: diff --git a/hshetl/loaders.py b/hshetl/loaders.py index 30b6017..e014b07 100644 --- a/hshetl/loaders.py +++ b/hshetl/loaders.py @@ -40,12 +40,14 @@ __all__ = ['AbstractLoader', 'CsvLoader', 'SqlAlchemyLoader', 'LdapLoader'] class AbstractLoader(object): '''A base class for loaders. - Construction: + .. function:: __init__([connector[, operations]]) - :param hshetl.connectors.AbstractConnector connector: - The connector this loader uses to write the data. - :param list operations: - The operations that will be performed. Possible values: ``insert``, ``update`` and ``delete`` + Constructor + + :param connector: The connector this loader uses to write the data. + :param operations: The operations that will be performed. Possible values: ``insert``, ``update`` and ``delete`` + :type connector: :class:`hshetl.connectors.AbstractConnector` or None + :type operations: list ''' @@ -105,12 +107,13 @@ class AbstractLoader(object): class CsvLoader(AbstractLoader): '''A loader that creates data in CSV format. - Construction: + .. function:: __init__([dialect[, **kwargs]]) + + Constructor - :param hshetl.Dialect dialect: - The CSV dialect that ill be used for CSV style. - :param dict ``**kwargs``: - Accepts parameters from :class:`.AbstractExtractor`. + :param dialect: The CSV dialect that will be used for CSV style. + :param **kwargs: Accepts parameters from :class:`.AbstractExtractor`. + :type dialect: :class:`hshetl.Dialect` YAML definition sample: @@ -179,12 +182,13 @@ class CsvLoader(AbstractLoader): class SqlAlchemyLoader(AbstractLoader): '''A loader for SQL data sources using the SQLAlchemyConnector. - Construction: + .. function:: __init__(table_name[, **kwargs]) - :param str table_name: - The name of the table in which to write the data. - :param dict ``**kwargs``: - Accepts parameters from :class:`.AbstractExtractor`. + Constructor + + :param table_name: The name of the table in which to write the data. + :param **kwargs: Accepts parameters from :class:`.AbstractExtractor`. + :type table_name: string YAML definition sample: @@ -271,16 +275,17 @@ class SqlAlchemyLoader(AbstractLoader): class LdapLoader(AbstractLoader): '''A loader for LDAP data sources using the ldap and ldap.modlist modules. - Construction: + .. function:: __init__(rdn, base, objectClass[, **kwargs]) + + Constructor - :param str rdn: - The attribute used to create a DN for new records. - :param str base: - The container in which new records will be created created. - :param str objectClass: - The objectClass for new records - use this instead of providing it inside your entity. - :param dict ``**kwargs``: - Accepts parameters from :class:`.AbstractExtractor`. + :param rdn: The attribute used to create a DN for new records. + :param base: The container in which new records will be created created. + :param objectClass: The objectClass for new records - use this instead of providing it inside your entity. + :param **kwargs: Accepts parameters from :class:`.AbstractExtractor`. + :type rdn: string + :type base: string + :type objectClass: string YAML definition sample: diff --git a/hshetl/transformers.py b/hshetl/transformers.py index b20d0b4..ca75a7d 100644 --- a/hshetl/transformers.py +++ b/hshetl/transformers.py @@ -32,12 +32,14 @@ from hshetl.exc import TransformerException class AbstractTransformer(object): '''A base class for transformers. - Construction: + .. function:: __init__([entity[, name]]) - :param hshetl.entity.Entity entity: - The entity this transformer works with. - :param unicode name: - The name of a transformer is usually used as name for the created container + Constructor + + :param entity: The entity this transformer works with. + :param name: The name of a transformer is usually used as name for the created container + :type entity: :class:`hshetl.entities.Entity` or None + :type name: unicode or None ''' @@ -79,16 +81,17 @@ class CompareTransformer(AbstractTransformer): This record will be mapped like the target system - as it is the comparison of changes when loading data from source into target. - Construction: + .. function:: __init__([source[, target[, mapping[, *args[, **kwargs]]]]]) + + Constructor - :param hshetl.entities.Container source: - The source container. - :param hshetl.entities.Container target: - The target container. - :param dict mapping: - If you do not want the result properties to be mapped like in target system, provide a mapping here. - :param dict ``**kwargs``: - Accepts parameters from :class:`.AbstractTransformer`. + :param source: The source container. + :param target: The target container. + :param mapping: If you do not want the result properties to be mapped like in target system, provide a mapping here. + :param **kwargs: Accepts parameters from :class:`.AbstractTransformer`. + :type source: :class:`hshetl.entities.Container` or None + :type target: :class:`hshetl.entities.Container` or None + :type mapping: dict or None YAML definition sample: @@ -166,6 +169,13 @@ class CompareTransformer(AbstractTransformer): class PropertyTransformer(object): '''Implements possible operations for the PropertiesTransformer. + .. function:: __init__([value]) + + Constructor + + :param value: initial value + :type value: mixed + Methodchaining is possible, which allows you to perform multiple transformations related to one property. @@ -203,14 +213,15 @@ class PropertiesTransformer(AbstractTransformer): Applies all operations to the fields in the new data container. - Construction: + .. function:: __init__([source[, operations[, *args[, **kwargs]]]]) + + Constructor - :param hshetl.entities.Container source: - The source container. - :param dict operations: - The operations that will be performed. The key is the property that should be transformed and the value a method call. - :param dict ``**kwargs``: - Accepts parameters from :class:`.AbstractTransformer`. + :param source: The source container. + :param operations: The operations that will be performed. The key is the property that should be transformed and the value a method call. + :param **kwargs: Accepts parameters from :class:`.AbstractTransformer`. + :type source: :class:`hshetl.entities.Container` or None + :type operations: dict All property values of the entity are available in the local scope of your method call. -- GitLab