[TASK] Introduces config dictionary

The config dictionary now acts as a place for
definitions of specific cases.
So now you can set the identifier if the object is
referencable or change the display type for desired
objects or create children for !!list types.

hshetl/editor.py

@@ -5,6 +5,52 @@ import json
 import re
 from hshetl.exc import *
 
+
+config = {
+            "TYPE_MAPPING": {
+                "string" : "!!str",
+                "unicode" : "!!str",
+                "int" : "!!int",
+                "bool" : "!!bool",
+                "boolean" : "!!bool",
+                "list" : "!!list",
+                "dict" : "!!map"
+            },
+            "TYPE_DISPLAY_MAPPING": {
+                "!!str" : "textfield",
+                "!!int" : "textfield",
+                "!!bool" : "checkbox",
+                "!!list" : "csv",
+                "!!map" : "table"
+            },
+            "SPECIFIC_TYPE_DISPLAY_MAPPING": {
+                "ExtractionJob.collision_handling": {
+                    "display" : "select", "str_choices": [
+                        "BREAKALL",
+                        "BREAKJOIN",
+                        "BREAKCONTAINER",
+                        "BREAKNEVER"
+                    ]
+                },
+                "JobList.jobs": "list"
+            },
+            "IDENTIFIERS": {
+                #"SQLiteConnector": "name"
+            },
+            "CHILDREN": {
+                "JobList.jobs": ["!bash", "!sqlquery", "!plsqlquery", "!extract", "!transform", "!inspect", "!load", "!sync", "!copy"]
+            },
+            "REFERENCABLE": [
+                "Entity",
+                "FileConnector",
+                "LdapConnector",
+                "OracleConnector",
+                "MySQLConnector",
+                "SQLiteConnector",
+                "PostgreSQLConnector"
+            ]
+          }
+
 def find_subclasses(cls):
     result = []
     for subcls in cls.__subclasses__():
@@ -15,11 +61,28 @@ def find_subclasses(cls):
     return result
 
 def apply_type_mapping(possible_types):
-    return possible_types
+    mapped_types = []
+    for p_type in possible_types:
+        if p_type in config["TYPE_MAPPING"].keys():
+            mapped_types.append(config["TYPE_MAPPING"][p_type])
+        else:
+            if p_type.startswith("hshetl."):
+                cls = eval(p_type)
+            elif p_type.endswith("Connector"):
+                cls = eval("hshetl.connectors." + p_type)
+            elif p_type.endswith("Extractor"):
+                cls = eval("hshetl.extractors." + p_type)
+            elif p_type.endswith("Loader"):
+                cls = eval("hshetl.loaders." + p_type)
+            else:
+                continue
+            if hasattr(cls, "yaml_tag"):
+                mapped_types.append(cls.yaml_tag)
+    return mapped_types
 
 def resolve_param_type(cls, property):
     possible_types = []
-    pattern = ":type " + property + ": (\\w*)(( or )?(\\w+)?)?"
+    pattern = ":type " + property + ": ((:class:\`(.*)`)|(\w*))(( or )?(\\w+)?)*"
     result = re.search(pattern, cls.__doc__)
     if result == None:
         for parent_cls in cls.__mro__:
@@ -33,20 +96,63 @@ def resolve_param_type(cls, property):
         if param_type == "" or param_type == None or " " in param_type:
             continue
         possible_types.append(param_type)
+        if "." in param_type:
+            for clas in itersubclasses(eval(param_type)):
+                if clas.__name__ not in possible_types:
+                    possible_types.append(clas.__name__)
+            if possible_types == []:
+                possible_types.append(param_type)
+        else:
+            if param_type not in possible_types:
+                possible_types.append(param_type)
     return apply_type_mapping(possible_types)
 
+def resolve_display_type(sup_type, property, cls_name):
+    if config["SPECIFIC_TYPE_DISPLAY_MAPPING"].has_key(cls_name + "." + property):
+        return config["SPECIFIC_TYPE_DISPLAY_MAPPING"][cls_name + "." + property]
+    elif config["TYPE_DISPLAY_MAPPING"].has_key(sup_type):
+        return config["TYPE_DISPLAY_MAPPING"][sup_type]
+    elif sup_type[1] != "!":
+        return "inline"
+    else:
+        return "textfield"
+
 def json_encode_class(cls):
     arg_resolver = hshetl.ConfigurationArgumentMatcher()
     props, required = arg_resolver.get_constructor_arguments(cls)
+    if cls.__name__ in config["REFERENCABLE"]:
+        cfg = {"yaml_tag": cls.yaml_tag,
+               "identifier": config["IDENTIFIERS"].get(cls.__name__, "name"),
+               "verbose_name": cls.__name__,
+               "properties": {}}
+    else:
         cfg = {"yaml_tag": cls.yaml_tag,
                "verbose_name": cls.__name__,
                "properties": {}}
     for prop in props:
         supported_types = resolve_param_type(cls, prop)
-        cfg["properties"][prop] = {"type": supported_types,
-                                   "form": {"display": "textfield"}}
-        if prop in required:
-            cfg["properties"][prop]["defaults"] = "foo"
+        param_list = []
+        for s_type in supported_types:
+            children = None
+            display_type = resolve_display_type(s_type, prop, cls.__name__)
+            if s_type == "!!list":
+                children = config["CHILDREN"][cls.__name__ + "." + prop] if config["CHILDREN"].has_key(cls.__name__ + "." + prop) else None
+
+            if children == None:
+                if isinstance(display_type, dict):
+                    param_list.append({"type": s_type, "form": display_type, "defaults": "foo"})
+                else:
+                    param_list.append({"type": s_type, "form": {"display": display_type}, "defaults": "foo"})
+            else:
+                if isinstance(display_type, dict):
+                    param_list.append({"type": s_type, "children": children, "form": display_type, "defaults": "foo"})
+                else:
+                    param_list.append({"type": s_type, "children": children, "form": {"display": display_type}, "defaults": "foo"})
+        try:
+            cfg["properties"][prop] = param_list if len(param_list) > 1 else param_list[0]
+        except Exception:
+            #this is not supposed to be here in the final version.
+            cfg["properties"][prop] = {"type" : None, "form": {"display": "textfield"}, "defaults": "foo"}
     return cfg
 
 def dump_schema():

hshetl/entities.py

@@ -503,7 +503,7 @@ class Result(object):
 
         :param entity: the entity to be representated
         :param source: system to fetch data from
-        :type entity: :class:`Entity`
+        :type entity: :class:`hshetl.entities.Entity`
         :type source: string
 
     A Result describes what records between the

hshetl/extractors.py

@@ -44,7 +44,7 @@ class AbstractExtractor(object):
         Constructor
 
         :param connector: connector instance
-        :type connector: :class:`.AbstractExtractor`
+        :type connector: :class:`hshetl.connectors.AbstractConnector`
 
     '''
 
@@ -96,7 +96,7 @@ class SqlAlchemyExtractor(AbstractExtractor):
         :param query: query that fetches data from the database
         :param **kwargs: accepts parameters from :class:`.AbstractExtractor`.
         :type query: string
-        :type connector: :class:`.AbstractExtractor`
+        :type connector: :class:`hshetl.connectors.OracleConnector` or :class:`hshetl.connectors.MySQLConnector` or :class:`hshetl.connectors.SQLiteConnector` or :class:`hshetl.connectors.PostgreSQLConnector`
 
     YAML definition sample:
 
@@ -156,6 +156,7 @@ class LdapExtractor(AbstractExtractor):
         :type ldap_filter: string or None
         :type attributes: list
         :type page_size: int or None
+        :type connector: :class:`hshetl.connectors.LdapConnector`
 
     YAML definition sample:
 
@@ -283,6 +284,7 @@ class CsvExtractor(AbstractExtractor):
         :param dialect: CSV dialect to be use for CSV style
         :param **kwargs: Accepts parameters from :class:`.AbstractExtractor`.
         :type dialect: :class:`hshetl.Dialect`
+        :type connector: :class:`hshetl.connectors.FileConnector`
 
     YAML definition sample:
 

hshetl/loaders.py

@@ -30,7 +30,7 @@ from uuid import uuid4 as uuid
 from ldap import modlist
 from sqlalchemy import Table, MetaData
 from hshetl import yamlify, NameResolver, Dialect
-from connectors import AbstractConnector, connector_repository
+from connectors import AbstractConnector, FileConnector, SqlAlchemyConnector, LdapConnector, connector_repository
 from hshetl.exc import LoaderException, ConfigurationException
 
 
@@ -114,6 +114,7 @@ class CsvLoader(AbstractLoader):
         :param dialect: The CSV dialect that will be used for CSV style.
         :param **kwargs: Accepts parameters from :class:`.AbstractExtractor`.
         :type dialect: :class:`hshetl.Dialect`
+        :type connector: :class:`hshetl.connectors.FileConnector`
 
     YAML definition sample:
 
@@ -189,6 +190,7 @@ class SqlAlchemyLoader(AbstractLoader):
         :param table_name: The name of the table in which to write the data.
         :param **kwargs: Accepts parameters from :class:`.AbstractLoader`.
         :type table_name: string
+        :type connector: :class:`hshetl.connectors.OracleConnector` or :class:`hshetl.connectors.MySQLConnector` or :class:`hshetl.connectors.SQLiteConnector` or :class:`hshetl.connectors.PostgreSQLConnector`
 
     YAML definition sample:
 
@@ -286,6 +288,7 @@ class LdapLoader(AbstractLoader):
         :type rdn: string
         :type base: string
         :type objectClass: string
+        :type connector: :class:`hshetl.connectors.LdapConnector`
 
     YAML definition sample: