diff --git a/hshetl/__init__.py b/hshetl/__init__.py index c37e00465eb7438057a614c44deb0ea1a5d743d5..bd49b071a20e9e4986d6d783788aa2128d7212f6 100644 --- a/hshetl/__init__.py +++ b/hshetl/__init__.py @@ -130,7 +130,22 @@ class ConfigurationArgumentMatcher(object): class NameResolver(object): - '''Decorator to resolve a reference from a repository.''' + '''Decorator to resolve a reference from a repository. + + .. function:: __init__(repository, argument_name[, argument_value[, add]]) + + Constructor + + :param repository: The repository from which to fetch the reference. + :param argument_name: The name of the argument. + :param argument_value: The value of the argument - use this if you use this class not as a decorator. + :param add: Indicates if the object should be added to the repository if no ref is given. + :type repository: :class:`AbstractRepository` + :type argument_name: string + :type argument_value: mixed + :type add: bool + + ''' def __init__(self, repository, argument_name, argument_value = None, add = False): '''Creates the decorator''' @@ -185,6 +200,13 @@ class AbstractRepository(OrderedDict, object): May act as singleton inside hshetl. + .. function:: __init__([items]) + + Constructor + + :param items: Item to be stored in the entity + :type items: list + ''' def __init__(self, items = []): @@ -248,23 +270,27 @@ class AbstractRepository(OrderedDict, object): class Dialect(csv.Dialect): '''A Dialect class for csv dialects - .. seealso:: - http://docs.python.org/2/library/csv.html#dialects-and-formatting-parameters - - :param str delimiter: - Default ',' - :param boolean doublequote: - Default True - :param str/False escapechar: - Default None - :param str lineterminator: - Default ``\\n`` - :param str quotechar: - Default '"' - :param boolean skipinitialspace: - Default False - :param int quoting: - Default 0 + .. seealso:: http://docs.python.org/2/library/csv.html#dialects-and-formatting-parameters + + .. function:: __init__([delimiter[, doubleqoute[, escapechar[, lineterminator[, quotechar[, skipinitialspace[, quoting]]]]]]]) + + Constructor + + :param delimiter: Default ',' + :param doublequote: Default True + :param escapechar: Default None + :param lineterminator: Default ``\\n`` + :param quotechar: Default '"' + :param skipinitialspace: Default False + :param quoting: Default 0 + + :type delimiter: string + :type doublequote: bool + :type escapechar: string or False + :type lineterminator: string + :type quotechar: string + :type skipinitialspace: bool + :type quoting: int Example yaml configuration: @@ -287,7 +313,7 @@ class Dialect(csv.Dialect): def __init__(self, delimiter=',', doublequote = True, - escapechar = None, + escapechar = False, lineterminator = '\n', quotechar = '"', skipinitialspace = False, diff --git a/hshetl/editor.py b/hshetl/editor.py index f4d180836926d20844853ca2efb26ce6d26151f4..3da163bbc1248acec75e21668437b0f94867af4b 100644 --- a/hshetl/editor.py +++ b/hshetl/editor.py @@ -2,6 +2,8 @@ from __future__ import print_function import hshetl import yaml import json +import re +from hshetl.exc import * def find_subclasses(cls): result = [] @@ -12,6 +14,27 @@ def find_subclasses(cls): result += find_subclasses(subcls) return result +def apply_type_mapping(possible_types): + return possible_types + +def resolve_param_type(cls, property): + possible_types = [] + pattern = ":type " + property + ": (\\w*)(( or )?(\\w+)?)?" + result = re.search(pattern, cls.__doc__) + if result == None: + for parent_cls in cls.__mro__: + if parent_cls == yaml.YAMLObject or parent_cls == object: + continue + result = re.search(pattern, parent_cls.__doc__) + if result != None: break + if result == None: + raise DocStringMissesTypeException("The property \"%s\" was not found in class %s or its parents." % (property, cls.__name__)) + for param_type in result.groups(): + if param_type == "" or param_type == None or " " in param_type: + continue + possible_types.append(param_type) + return apply_type_mapping(possible_types) + def json_encode_class(cls): arg_resolver = hshetl.ConfigurationArgumentMatcher() props, required = arg_resolver.get_constructor_arguments(cls) @@ -19,7 +42,8 @@ def json_encode_class(cls): "verbose_name": cls.__name__, "properties": {}} for prop in props: - cfg["properties"][prop] = {"type": "!!str", + supported_types = resolve_param_type(cls, prop) + cfg["properties"][prop] = {"type": supported_types, "form": {"display": "textfield"}} if prop in required: cfg["properties"][prop]["defaults"] = "foo" @@ -36,12 +60,12 @@ def dump_schema(): "jobs": []}} classes = [] for cls in yaml_subclasses.itervalues(): + if cls.yaml_tag == "!void": + continue if "job" in cls.__name__.lower(): docs["children"]["jobs"].append(cls.yaml_tag) elif "connector" in cls.__name__.lower(): docs["children"]["connectors"].append(cls.yaml_tag) - if cls.yaml_tag != "!void": - classes.append(json_encode_class(cls)) with open('editor_data.js','w') as f: print("\"use strict\";\n\nvar hshetl_editor_data = " + json.dumps({"documents": docs, "classes": classes}, diff --git a/hshetl/entities.py b/hshetl/entities.py index fdc95876cdaa4f6fa4040ce9d00b5888f12f84cd..27ed6ca1f46df3887f21405cc5f5849d76ca906c 100644 --- a/hshetl/entities.py +++ b/hshetl/entities.py @@ -47,6 +47,7 @@ class Entity(AbstractRepository): :type name: string :type join_properties: list :type containers: list + :type items: list The entity stores information about the properties, their data types and names in the different containers. It diff --git a/hshetl/exc.py b/hshetl/exc.py index 7fb62f3aac9573b5267bd3c9ac54b469e0e42f17..af0ef6e972172bbf8d780b6b3dc425f03ce763b7 100644 --- a/hshetl/exc.py +++ b/hshetl/exc.py @@ -86,3 +86,6 @@ class UnexpectedEnvironmentException(Exception): class EscapeDatetimeException(Exception): pass + +class DocStringMissesTypeException(Exception): + pass diff --git a/hshetl/extractors.py b/hshetl/extractors.py index 4afe88c54d356cdd8dd1c80db9dfa9c84d87fb6e..3c573a8d99064d91744219a9445d00c34cdeacaa 100644 --- a/hshetl/extractors.py +++ b/hshetl/extractors.py @@ -96,6 +96,7 @@ class SqlAlchemyExtractor(AbstractExtractor): :param query: query that fetches data from the database :param **kwargs: accepts parameters from :class:`.AbstractExtractor`. :type query: string + :type connector: :class:`.AbstractExtractor` YAML definition sample: diff --git a/hshetl/jobs.py b/hshetl/jobs.py index e69ef248ae5abb5bb8b0c22bb13d96dba16b5564..2121cd5e07cb686620a4c8d4754872f915ee0b4a 100644 --- a/hshetl/jobs.py +++ b/hshetl/jobs.py @@ -775,6 +775,10 @@ class CopyJob(JobList, EntityJob): :param inspect: If ``True`` an InspectJob is added after the :class:`.TransformerJob` :param **kwargs: Accepts parameters from :class:`.EntityJob` and :class:`.JobList`. + :type source: dict + :type target: dict + :type inspect: bool + YAML sample definition: .. code-block:: yaml diff --git a/hshetl/loaders.py b/hshetl/loaders.py index e014b07e6cebba599290c7de2aa6576c6b58c238..6d8c1c6eaf2cc3104344a16410355eb905d84292 100644 --- a/hshetl/loaders.py +++ b/hshetl/loaders.py @@ -187,7 +187,7 @@ class SqlAlchemyLoader(AbstractLoader): Constructor :param table_name: The name of the table in which to write the data. - :param **kwargs: Accepts parameters from :class:`.AbstractExtractor`. + :param **kwargs: Accepts parameters from :class:`.AbstractLoader`. :type table_name: string YAML definition sample: