diff --git a/hshetl/editor.py b/hshetl/editor.py
index 3da163bbc1248acec75e21668437b0f94867af4b..42415c1ede6787bbda8bb98c64114faacd6c8e63 100644
--- a/hshetl/editor.py
+++ b/hshetl/editor.py
@@ -5,6 +5,52 @@ import json
import re
from hshetl.exc import *
+
+config = {
+ "TYPE_MAPPING": {
+ "string" : "!!str",
+ "unicode" : "!!str",
+ "int" : "!!int",
+ "bool" : "!!bool",
+ "boolean" : "!!bool",
+ "list" : "!!list",
+ "dict" : "!!map"
+ },
+ "TYPE_DISPLAY_MAPPING": {
+ "!!str" : "textfield",
+ "!!int" : "textfield",
+ "!!bool" : "checkbox",
+ "!!list" : "csv",
+ "!!map" : "table"
+ },
+ "SPECIFIC_TYPE_DISPLAY_MAPPING": {
+ "ExtractionJob.collision_handling": {
+ "display" : "select", "str_choices": [
+ "BREAKALL",
+ "BREAKJOIN",
+ "BREAKCONTAINER",
+ "BREAKNEVER"
+ ]
+ },
+ "JobList.jobs": "list"
+ },
+ "IDENTIFIERS": {
+ #"SQLiteConnector": "name"
+ },
+ "CHILDREN": {
+ "JobList.jobs": ["!bash", "!sqlquery", "!plsqlquery", "!extract", "!transform", "!inspect", "!load", "!sync", "!copy"]
+ },
+ "REFERENCABLE": [
+ "Entity",
+ "FileConnector",
+ "LdapConnector",
+ "OracleConnector",
+ "MySQLConnector",
+ "SQLiteConnector",
+ "PostgreSQLConnector"
+ ]
+ }
+
def find_subclasses(cls):
result = []
for subcls in cls.__subclasses__():
@@ -15,11 +61,28 @@ def find_subclasses(cls):
return result
def apply_type_mapping(possible_types):
- return possible_types
+ mapped_types = []
+ for p_type in possible_types:
+ if p_type in config["TYPE_MAPPING"].keys():
+ mapped_types.append(config["TYPE_MAPPING"][p_type])
+ else:
+ if p_type.startswith("hshetl."):
+ cls = eval(p_type)
+ elif p_type.endswith("Connector"):
+ cls = eval("hshetl.connectors." + p_type)
+ elif p_type.endswith("Extractor"):
+ cls = eval("hshetl.extractors." + p_type)
+ elif p_type.endswith("Loader"):
+ cls = eval("hshetl.loaders." + p_type)
+ else:
+ continue
+ if hasattr(cls, "yaml_tag"):
+ mapped_types.append(cls.yaml_tag)
+ return mapped_types
def resolve_param_type(cls, property):
possible_types = []
- pattern = ":type " + property + ": (\\w*)(( or )?(\\w+)?)?"
+ pattern = ":type " + property + ": ((:class:\`(.*)`)|(\w*))(( or )?(\\w+)?)*"
result = re.search(pattern, cls.__doc__)
if result == None:
for parent_cls in cls.__mro__:
@@ -33,20 +96,63 @@ def resolve_param_type(cls, property):
if param_type == "" or param_type == None or " " in param_type:
continue
possible_types.append(param_type)
+ if "." in param_type:
+ for clas in itersubclasses(eval(param_type)):
+ if clas.__name__ not in possible_types:
+ possible_types.append(clas.__name__)
+ if possible_types == []:
+ possible_types.append(param_type)
+ else:
+ if param_type not in possible_types:
+ possible_types.append(param_type)
return apply_type_mapping(possible_types)
+def resolve_display_type(sup_type, property, cls_name):
+ if config["SPECIFIC_TYPE_DISPLAY_MAPPING"].has_key(cls_name + "." + property):
+ return config["SPECIFIC_TYPE_DISPLAY_MAPPING"][cls_name + "." + property]
+ elif config["TYPE_DISPLAY_MAPPING"].has_key(sup_type):
+ return config["TYPE_DISPLAY_MAPPING"][sup_type]
+ elif sup_type[1] != "!":
+ return "inline"
+ else:
+ return "textfield"
+
def json_encode_class(cls):
arg_resolver = hshetl.ConfigurationArgumentMatcher()
props, required = arg_resolver.get_constructor_arguments(cls)
- cfg = {"yaml_tag": cls.yaml_tag,
- "verbose_name": cls.__name__,
- "properties": {}}
+ if cls.__name__ in config["REFERENCABLE"]:
+ cfg = {"yaml_tag": cls.yaml_tag,
+ "identifier": config["IDENTIFIERS"].get(cls.__name__, "name"),
+ "verbose_name": cls.__name__,
+ "properties": {}}
+ else:
+ cfg = {"yaml_tag": cls.yaml_tag,
+ "verbose_name": cls.__name__,
+ "properties": {}}
for prop in props:
supported_types = resolve_param_type(cls, prop)
- cfg["properties"][prop] = {"type": supported_types,
- "form": {"display": "textfield"}}
- if prop in required:
- cfg["properties"][prop]["defaults"] = "foo"
+ param_list = []
+ for s_type in supported_types:
+ children = None
+ display_type = resolve_display_type(s_type, prop, cls.__name__)
+ if s_type == "!!list":
+ children = config["CHILDREN"][cls.__name__ + "." + prop] if config["CHILDREN"].has_key(cls.__name__ + "." + prop) else None
+
+ if children == None:
+ if isinstance(display_type, dict):
+ param_list.append({"type": s_type, "form": display_type, "defaults": "foo"})
+ else:
+ param_list.append({"type": s_type, "form": {"display": display_type}, "defaults": "foo"})
+ else:
+ if isinstance(display_type, dict):
+ param_list.append({"type": s_type, "children": children, "form": display_type, "defaults": "foo"})
+ else:
+ param_list.append({"type": s_type, "children": children, "form": {"display": display_type}, "defaults": "foo"})
+ try:
+ cfg["properties"][prop] = param_list if len(param_list) > 1 else param_list[0]
+ except Exception:
+ #this is not supposed to be here in the final version.
+ cfg["properties"][prop] = {"type" : None, "form": {"display": "textfield"}, "defaults": "foo"}
return cfg
def dump_schema():
diff --git a/hshetl/entities.py b/hshetl/entities.py
index 27ed6ca1f46df3887f21405cc5f5849d76ca906c..ba3a422d70452bebb34c05c10b5a95fc6d3a67c1 100644
--- a/hshetl/entities.py
+++ b/hshetl/entities.py
@@ -503,7 +503,7 @@ class Result(object):
:param entity: the entity to be representated
:param source: system to fetch data from
- :type entity: :class:`Entity`
+ :type entity: :class:`hshetl.entities.Entity`
:type source: string
A Result describes what records between the
diff --git a/hshetl/extractors.py b/hshetl/extractors.py
index 3c573a8d99064d91744219a9445d00c34cdeacaa..6ff0ecc3227452ca9e33a00b85d51a8d74c6201e 100644
--- a/hshetl/extractors.py
+++ b/hshetl/extractors.py
@@ -44,7 +44,7 @@ class AbstractExtractor(object):
Constructor
:param connector: connector instance
- :type connector: :class:`.AbstractExtractor`
+ :type connector: :class:`hshetl.connectors.AbstractConnector`
'''
@@ -96,7 +96,7 @@ class SqlAlchemyExtractor(AbstractExtractor):
:param query: query that fetches data from the database
:param **kwargs: accepts parameters from :class:`.AbstractExtractor`.
:type query: string
- :type connector: :class:`.AbstractExtractor`
+ :type connector: :class:`hshetl.connectors.OracleConnector` or :class:`hshetl.connectors.MySQLConnector` or :class:`hshetl.connectors.SQLiteConnector` or :class:`hshetl.connectors.PostgreSQLConnector`
YAML definition sample:
@@ -156,6 +156,7 @@ class LdapExtractor(AbstractExtractor):
:type ldap_filter: string or None
:type attributes: list
:type page_size: int or None
+ :type connector: :class:`hshetl.connectors.LdapConnector`
YAML definition sample:
@@ -283,6 +284,7 @@ class CsvExtractor(AbstractExtractor):
:param dialect: CSV dialect to be use for CSV style
:param **kwargs: Accepts parameters from :class:`.AbstractExtractor`.
:type dialect: :class:`hshetl.Dialect`
+ :type connector: :class:`hshetl.connectors.FileConnector`
YAML definition sample:
diff --git a/hshetl/loaders.py b/hshetl/loaders.py
index 6d8c1c6eaf2cc3104344a16410355eb905d84292..325b20739a04b0ebcd4eec81408d771b4da9ce4e 100644
--- a/hshetl/loaders.py
+++ b/hshetl/loaders.py
@@ -30,7 +30,7 @@ from uuid import uuid4 as uuid
from ldap import modlist
from sqlalchemy import Table, MetaData
from hshetl import yamlify, NameResolver, Dialect
-from connectors import AbstractConnector, connector_repository
+from connectors import AbstractConnector, FileConnector, SqlAlchemyConnector, LdapConnector, connector_repository
from hshetl.exc import LoaderException, ConfigurationException
@@ -114,6 +114,7 @@ class CsvLoader(AbstractLoader):
:param dialect: The CSV dialect that will be used for CSV style.
:param **kwargs: Accepts parameters from :class:`.AbstractExtractor`.
:type dialect: :class:`hshetl.Dialect`
+ :type connector: :class:`hshetl.connectors.FileConnector`
YAML definition sample:
@@ -189,6 +190,7 @@ class SqlAlchemyLoader(AbstractLoader):
:param table_name: The name of the table in which to write the data.
:param **kwargs: Accepts parameters from :class:`.AbstractLoader`.
:type table_name: string
+ :type connector: :class:`hshetl.connectors.OracleConnector` or :class:`hshetl.connectors.MySQLConnector` or :class:`hshetl.connectors.SQLiteConnector` or :class:`hshetl.connectors.PostgreSQLConnector`
YAML definition sample:
@@ -286,6 +288,7 @@ class LdapLoader(AbstractLoader):
:type rdn: string
:type base: string
:type objectClass: string
+ :type connector: :class:`hshetl.connectors.LdapConnector`
YAML definition sample: