Add new “salt.formulas” state to manage salt formulas

This state downloads formulas from git repositories and ensures
that they are recorded in the file_roots setting.

README.rst

@@ -61,6 +61,39 @@ official documentation
 
 Undo the effects of ``salt.pkgrepo``.
 
+``salt.formulas``
+-----------------
+
+Clone selected `Salt formulas
+<http://docs.saltstack.com/en/latest/topics/development/conventions/formulas.html>`_
+Git repositories under ``/srv/formulas`` and makes them available in the
+relevant ``file_roots`` settings. Pillar data can be used to customize all
+paths, URLs, etc.
+
+Here's a minimal pillar sample installing two formulas in the base
+environment.
+
+::
+
+    salt_formulas:
+      list:
+        base:
+          - salt-formula
+          - openssh-formula
+
+See pillar.example for an exhaustive list of settings available via pillar. Note
+that by default this state:
+
+- downloads the latest formulas from the `saltstack-formulas project
+  <https://github.com/saltstack-formulas>`_ on GitHub.
+- does not update the local repositories after the initial clone.
+  This is a safety measure since you do not control how the official
+  repositories evolve.
+
+If you configure the state to download the formulas from repositories that
+you control, then you can safely enable the
+``salt_formulas:git_opts:default:update`` pillar setting to ``True``.
+
 ``Configuration``
 =================
 Every option available in the templates can be set in pillar. Settings under 'salt' will be overridden by more specific settings under ``salt['master']``, ``salt['minion']`` or ``salt['cloud']``

pillar.example

@@ -73,3 +73,41 @@ salt_cloud_certs:
       -----BEGIN RSA PRIVATE KEY-----
       ...........
       -----END RSA PRIVATE KEY-----
+
+salt_formulas:
+  git_opts:
+    # The Git options can be customized differently for each
+    # environment, if an option is missing in a given environment, the
+    # value from "default" is used instead.
+    default:
+      # URL where the formulas git repositories are downloaded from
+      # it will be suffixed with <formula-name>.git
+      baseurl: https://github.com/saltstack-formulas
+      # Directory where Git repositories are downloaded
+      basedir: /srv/formulas
+      # Update the git repository to the latest version (False by default)
+      update: False
+      # Options passed directly to the git.latest state
+      options:
+        rev: master
+    dev:
+      basedir: /srv/formulas/dev
+      update: True
+      options:
+        rev: develop
+  # Options of the file.directory state that creates the directory where
+  # the git repositories of the formulas are stored
+  basedir_opts:
+    makedirs: True
+    user: root
+    group: root
+    mode: 755
+  # List of formulas to enable in each environment
+  list:
+    base:
+      - salt-formula
+      - postfix-formula
+    dev:
+      - salt-formula
+      - postfix-formula
+      - openssh-formula

salt/files/master.d/_defaults.conf

@@ -12,6 +12,7 @@
 #{{ configname }}: {{ default_value }}
 {%- endif -%}
 {%- endmacro -%}
+{%- from 'salt/formulas.jinja' import file_roots, formulas -%}
 ##### Primary configuration settings #####
 ##########################################
 # This configuration file is used to manage the behavior of the Salt Master
@@ -473,22 +474,12 @@ client_acl_blacklist:
 #     - /srv/salt/prod/states
 
 {% if 'file_roots' in master -%}
-file_roots:
-{%- for name, roots in master['file_roots']|dictsort %}
-  {{ name }}:
-{%- for dir in roots %}
-    - {{ dir }}
-{%- endfor -%}
-{%- endfor -%}
-{% elif 'file_roots' in salt -%}
-file_roots:
-{%- for name, roots in salt['file_roots']|dictsort %}
-  {{ name }}:
-{%- for dir in roots %}
-    - {{ dir }}
-{%- endfor -%}
-{%- endfor -%}
-{% else -%}
+{{ file_roots(master['file_roots']) }}
+{%- elif 'file_roots' in salt -%}
+{{ file_roots(salt['file_roots']) }}
+{%- elif formulas|length -%}
+{{ file_roots({'base': ['/srv/salt']}) }}
+{%- else -%}
 #file_roots:
 #  base:
 #    - /srv/salt

salt/files/minion.d/_defaults.conf

@@ -12,6 +12,7 @@
 #{{ configname }}: {{ default_value }}
 {%- endif -%}
 {%- endmacro -%}
+{%- from 'salt/formulas.jinja' import file_roots, formulas -%}
 ##### Primary configuration settings #####
 ##########################################
 
@@ -408,22 +409,12 @@ file_client: local
 #     - /srv/salt/prod/states
 #
 {% if 'file_roots' in minion -%}
-file_roots:
-{%- for name, roots in minion['file_roots']|dictsort %}
-  {{ name }}:
-{%- for dir in roots %}
-    - {{ dir }}
-{%- endfor -%}
-{%- endfor -%}
-{% elif 'file_roots' in salt -%}
-file_roots:
-{%- for name, roots in salt['file_roots']|dictsort %}
-  {{ name }}:
-{%- for dir in roots %}
-    - {{ dir }}
-{%- endfor -%}
-{%- endfor -%}
-{% else -%}
+{{ file_roots(minion['file_roots']) }}
+{%- elif 'file_roots' in salt -%}
+{{ file_roots(salt['file_roots']) }}
+{%- elif formulas|length -%}
+{{ file_roots({'base': ['/srv/salt']}) }}
+{%- else -%}
 #file_roots:
 #  base:
 #    - /srv/salt

salt/formulas.jinja

+{% set defaults = {
+    'baseurl': 'https://github.com/saltstack-formulas',
+    'basedir': '/srv/formulas',
+    'update': False,
+    'options': {},
+   }
+%}
+{% set formulas = salt['pillar.get']('salt_formulas:list', {}) %}
+
+{%- macro formulas_git_opt(env, opt) -%}
+{%- set value = salt['pillar.get']('salt_formulas:git_opts:{}:{}'.format(env, opt),
+     salt['pillar.get']('salt_formulas:git_opts:default:{}'.format(opt),
+       defaults[opt])) -%}
+{%- if value is mapping -%}
+{{ value|yaml }}
+{%- else -%}
+{{ value }}
+{%- endif -%}
+{%- endmacro -%}
+
+{%- macro formulas_roots(env) -%}
+{%- set value = [] -%}
+{%- for dir in formulas.get(env, []) -%}
+{%- do value.append('{}/{}'.format(formulas_git_opt(env, 'basedir'), dir)) -%}
+{%- endfor -%}
+{{ value|yaml }}
+{%- endmacro -%}
+
+{# Generate file_roots config merging standard salt config and list of
+   enabled formulas #}
+{%- macro file_roots(input) -%}
+{%- set processed_envs = [] -%}
+file_roots:
+{%- for name, roots in input|dictsort -%}
+{%- do processed_envs.append(name) %}
+  {{ name }}:
+{%- for dir in roots %}
+    - {{ dir }}
+{%- endfor -%}
+{%- for dir in formulas_roots(name)|load_yaml %}
+    - {{ dir }}
+{%- endfor -%}
+{%- endfor -%}
+{%- for name in formulas -%}
+{%- if name not in processed_envs %}
+  {{ name }}:
+{%- for dir in formulas_roots(name)|load_yaml %}
+    - {{ dir }}
+{%- endfor -%}
+{%- endif -%}
+{%- endfor -%}
+{%- endmacro -%}

salt/formulas.sls

+{% set processed_gitdirs = [] %}
+{% set processed_basedirs = [] %}
+
+{% from "salt/formulas.jinja" import formulas_git_opt with context %}
+
+# Loop over all formulas listed in pillar data
+{% for env, entries in salt['pillar.get']('salt_formulas:list').iteritems() %}
+{% for entry in entries %}
+
+{% set basedir = formulas_git_opt(env, 'basedir') %}
+{% set gitdir = '{}/{}'.format(basedir, entry) %}
+{% set update = formulas_git_opt(env, 'update')|load_yaml %}
+
+# Setup the directory hosting the Git repository
+{% if basedir not in processed_basedirs %}
+{% do processed_basedirs.append(basedir) %}
+{{ basedir }}:
+  file.directory:
+    {%- for key, value in salt['pillar.get']('salt_formulas:basedir_opts',
+                                             {'makedirs': True}).iteritems() %}
+    - {{ key }}: {{ value }}
+    {%- endfor %}
+{% endif %}
+
+# Setup the formula Git repository
+{% if gitdir not in processed_gitdirs %}
+{% do processed_gitdirs.append(gitdir) %}
+{% set options = formulas_git_opt(env, 'options')|load_yaml %}
+{{ gitdir }}:
+  git.latest:
+    - name: {{ formulas_git_opt(env, 'baseurl') }}/{{ entry }}.git
+    - target: {{ gitdir }}
+    {%- for key, value in options.iteritems() %}
+    - {{ key }}: {{ value }}
+    {%- endfor %}
+    - require:
+      - file: {{ basedir }}
+    {%- if not update %}
+    - unless: test -e {{ gitdir }}
+    {%- endif %}
+{% endif %}
+
+{% endfor %}
+{% endfor %}