feat: use `semantic-release` cross-formula standard structure

* Semi-automated using https://github.com/myii/ssf-formula/pull/31 * Includes: - Update TOFS - Use `bin/kitchen` - Use `dist: bionic` in Travis - Add `yamllint` and apply rules - Use `pillars_from_files` throughout - Replace EOL images in Kitchen & Travis - Add `develop` images in Kitchen & Travis * Fix (or ignore) errors shown below: ```bash salt-formula$ yamllint -s . ./pillar.example 1:1 warning missing document start "---" (document-start) 5:23 warning truthy value should be one of [false, true] (truthy) 8:25 warning truthy value should be one of [false, true] (truthy) 11:25 warning truthy value should be one of [false, true] (truthy) 19:21 warning truthy value should be one of [false, true] (truthy) 40:89 error line too long (108 > 88 characters) (line-length) 41:89 error line too long (112 > 88 characters) (line-length) 43:89 error line too long (112 > 88 characters) (line-length) 45:89 error line too long (110 > 88 characters) (line-length) 47:89 error line too long (89 > 88 characters) (line-length) 74:27 warning truthy value should be one of [false, true] (truthy) 82:9 error wrong indentation: expected 10 but found 8 (indentation) 101:14 warning truthy value should be one of [false, true] (truthy) 102:20 warning truthy value should be one of [false, true] (truthy) 103:89 error line too long (119 > 88 characters) (line-length) 121:7 warning comment not indented like content (comments-indentation) 122:24 error syntax error: found character '%' that cannot start any token 310:89 error line too long (102 > 88 characters) (line-length) 330:89 error line too long (113 > 88 characters) (line-length) 433:1 error too many blank lines (1 > 0) (empty-lines) ./salt/osmap.yaml 4:2 error syntax error: found character '%' that cannot start any token 6:89 error line too long (93 > 88 characters) (line-length) 22:89 error line too long (137 > 88 characters) (line-length) 23:89 error line too long (134 > 88 characters) (line-length) 33:89 error line too long (149 > 88 characters) (line-length) 34:89 error line too long (146 > 88 characters) (line-length) ./salt/osfamilymap.yaml 4:2 error syntax error: found character '%' that cannot start any token 6:89 error line too long (94 > 88 characters) (line-length) 24:89 error line too long (149 > 88 characters) (line-length) 25:89 error line too long (146 > 88 characters) (line-length) 39:89 error line too long (105 > 88 characters) (line-length) 40:89 error line too long (127 > 88 characters) (line-length) 56:89 error line too long (101 > 88 characters) (line-length) ./salt/ospyvermap.yaml 4:1 warning missing document start "---" (document-start) ./salt/defaults.yaml 3:1 warning missing document start "---" (document-start) 7:21 warning truthy value should be one of [false, true] (truthy) 8:12 warning truthy value should be one of [false, true] (truthy) 9:23 warning truthy value should be one of [false, true] (truthy) 10:19 warning truthy value should be one of [false, true] (truthy) 14:25 warning truthy value should be one of [false, true] (truthy) 15:25 warning truthy value should be one of [false, true] (truthy) 16:27 warning truthy value should be one of [false, true] (truthy) 17:27 warning truthy value should be one of [false, true] (truthy) 39:28 warning truthy value should be one of [false, true] (truthy) 41:28 warning truthy value should be one of [false, true] (truthy) 45:24 warning truthy value should be one of [false, true] (truthy) 49:30 warning truthy value should be one of [false, true] (truthy) 54:28 warning truthy value should be one of [false, true] (truthy) 63:25 warning truthy value should be one of [false, true] (truthy) 68:15 warning truthy value should be one of [false, true] (truthy) ```

feat: use `semantic-release` cross-formula standard structure
ebfeba2c · Imran Iqbal · b59500cf · ebfeba2c · ebfeba2c · ebfeba2c
Unverified Commit ebfeba2c authored 5 years ago by Imran Iqbal
--- a/.gitignore
+++ b/.gitignore
@@ -48,6 +48,7 @@ coverage.xml
 .kitchen
 .kitchen.local.yml
 kitchen.local.yml
+junit-*.xml
 # Translations
 *.mo
@@ -110,5 +111,12 @@ Gemfile.lock
 docs/*.md
 # Vim
-*.swp
+*.sw?
-*.swo
+## Collected when centralising formulas (check and sort)
+# `collectd-formula`
+.pytest_cache/
+/.idea/
+Dockerfile.*_*
+ignore/
+tmp/
--- a/.travis.yml
+++ b/.travis.yml
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+---
+dist: bionic
 stages:
  - test
-  - commitlint
+  - lint
  - name: release
    if: branch = master AND type != pull_request
@@ -15,36 +19,57 @@ services:
 # the `platforms` defined in `kitchen.yml`
 env:
  matrix:
-    - INSTANCE: v2019-2-py3-debian-9
+    # The ordering used below has been selected based on the time required in Travis
-    - INSTANCE: v2019-2-py3-ubuntu-1804
+    # The slower ones are kept as high up as possible, to run concurrently rather than
-    # - INSTANCE: v2019-2-py2-centos-7
+    # slow down the entire run at the end (i.e. `centos-6` and `opensuse`)
-    - INSTANCE: v2019-2-py2-fedora-29
+    # However, the groupings needed to be maintained in some semblance of order
+    # so this is a best-effort matrix, in the circumstances
-    - INSTANCE: v2018-3-py2-debian-8
+    # develop-py3
-    - INSTANCE: v2018-3-py2-ubuntu-1604
+    - INSTANCE: develop-py3-debian-9
-    - INSTANCE: v2018-3-py2-bootstrap-centos-6
+    - INSTANCE: develop-py3-opensuse-150
-    - INSTANCE: v2018-3-py2-forced-version-fedora-28
+    # - INSTANCE: develop-py3-ubuntu-1804
-    # - INSTANCE: v2018-3-py2-opensuse-423
-    - INSTANCE: v2017-7-py2-debian-8
+    # 2017.7-py2
-    - INSTANCE: v2017-7-py2-ubuntu-1604
    - INSTANCE: v2017-7-py2-bootstrap-centos-6
+    # - INSTANCE: v2017-7-py2-debian-8
+    - INSTANCE: v2017-7-py2-ubuntu-1604
+    # 2018.3-py2
+    # - INSTANCE: v2018-3-py2-centos-7
+    - INSTANCE: v2018-3-py2-debian-8
+    - INSTANCE: v2018-3-py2-forced-version-fedora-29
+    # - INSTANCE: v2018-3-py2-ubuntu-1604
+    # 2019.2-py2
+    - INSTANCE: v2019-2-py2-centos-7
+    - INSTANCE: v2019-2-py2-fedora-30
+    # 2019.2-py3
+    # - INSTANCE: v2019-2-py3-debian-9
+    - INSTANCE: v2019-2-py3-opensuse-150
+    - INSTANCE: v2019-2-py3-ubuntu-1804
 script:
-  - bundle exec kitchen verify ${INSTANCE}
+  - bin/kitchen verify ${INSTANCE}
 jobs:
  include:
-    # Define the commitlint stage
+    # Define the `lint` stage (runs `yamllint` and `commitlint`)
-    - stage: commitlint
+    - stage: lint
      language: node_js
      node_js: lts/*
      before_install: skip
      script:
+        # Install and run `yamllint`
+        # Need at least `v1.17.0` for the `yaml-files` setting
+        - pip install --user yamllint>=1.17.0
+        - yamllint -s .
+        # Install and run `commitlint`
        - npm install @commitlint/config-conventional -D
        - npm install @commitlint/travis-cli -D
        - commitlint-travis
-    # Define the release stage that runs semantic-release
+    # Define the release stage that runs `semantic-release`
    - stage: release
      language: node_js
      node_js: lts/*

--- a/.yamllint
+++ b/.yamllint
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+---
+# Extend the `default` configuration provided by `yamllint`
+extends: default
+# Files to ignore completely
+# 1. All YAML files under directory `node_modules/`, introduced during the Travis run
+# 2. Any SLS files under directory `test/`, which are actually state files
+# 3. All YAML files heavily reliant on Jinja; these can be tackled in a subsequent PR
+ignore: |
+  node_modules/
+  test/**/states/**/*.sls
+  salt/osfamilymap.yaml
+  salt/osmap.yaml
+yaml-files:
+  # Default settings
+  - '*.yaml'
+  - '*.yml'
+  - .yamllint
+  # SaltStack Formulas additional settings
+  - '*.example'
+  - test/**/*.sls
+rules:
+  empty-values:
+    forbid-in-block-mappings: true
+    forbid-in-flow-mappings: true
+  line-length:
+    # Increase from default of `80`
+    # Based on https://github.com/PyCQA/flake8-bugbear#opinionated-warnings (`B950`)
+    max: 88
--- a/FORMULA
+++ b/FORMULA
@@ -6,3 +6,4 @@ release: 1
 minimum_version: 2015.8
 summary: Formula for install Saltstack
 description: Formula for installing Saltstack
+top_level_dir: salt
--- a/bin/kitchen
+++ b/bin/kitchen
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+#
+# This file was generated by Bundler.
+#
+# The application 'kitchen' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+require "rubygems"
+require "bundler/setup"
+load Gem.bin_path("test-kitchen", "kitchen")
--- a/docs/CONTRIBUTING.rst
+++ b/docs/CONTRIBUTING.rst
@@ -157,5 +157,3 @@ An example of that:
   `template package`, this formula no longer supports the installation of
   packages.
--- a/docs/README.rst
+++ b/docs/README.rst
@@ -205,28 +205,42 @@ Testing
 Linux testing is done with ``kitchen-salt``.
-``kitchen converge``
+Requirements
-^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^
+* Ruby
+* Docker
+.. code-block:: bash
+   $ gem install bundler
+   $ bundle install
+   $ bin/kitchen test [platform]
+Where ``[platform]`` is the platform name defined in ``kitchen.yml``,
+e.g. ``debian-9-2019-2-py3``.
+``bin/kitchen converge``
+^^^^^^^^^^^^^^^^^^^^^^^^
-Creates the docker instance and runs the ``template`` main state, ready for testing.
+Creates the docker instance and runs the ``salt`` main states, ready for testing.
-``kitchen verify``
+``bin/kitchen verify``
-^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^
 Runs the ``inspec`` tests on the actual instance.
-``kitchen destroy``
+``bin/kitchen destroy``
-^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^
 Removes the docker instance.
-``kitchen test``
+``bin/kitchen test``
-^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^
 Runs all of the stages above in one go: i.e. ``destroy`` + ``converge`` + ``verify`` + ``destroy``.
-``kitchen login``
+``bin/kitchen login``
-^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^
 Gives you SSH access to the instance for manual testing.
--- a/docs/TOFS_pattern.rst
+++ b/docs/TOFS_pattern.rst
@@ -39,7 +39,7 @@ Using SaltStack is a simple and effective way to implement configuration managem
 To avoid this situation we can use the `pillar mechanism <http://docs.saltstack.com/en/latest/topics/pillar/>`_, which is designed to provide controlled access to data from the minions based on some selection rules. As pillar data could be easily integrated in the `Jinja <http://docs.saltstack.com/en/latest/topics/tutorials/pillar.html>`_ templates, it is a good mechanism to store values to be used in the final rendering of state files and templates.
-There are a variety of approaches on the usage of pillar and templates as seen in the `saltstack-formulas <https://github.com/saltstack-formulas>`_' repositories. `Some <https://github.com/saltstack-formulas/nginx-formula/pull/18>`_ `developments <https://github.com/saltstack-formulas/php-formula/pull/14>`_ stress the initial purpose of pillar data into a storage for most of the possible variables for a determined system configuration. This, in my opinion, is shifting too much load from the original template files approach. Adding up some `non-trivial Jinja <https://github.com/spsoit/nginx-formula/blob/81de880fe0276dd9488ffa15bc78944c0fc2b919/nginx/ng/files/nginx.conf>`_ code as essential part of composing the state file definitely makes SaltStack state files (hence formulas) more difficult to read. The extreme of this approach is that we could end up with a new render mechanism, implemented in Jinja, storing everything needed in pillar data to compose configurations. Additionally, we are establishing a strong dependency with the Jinja renderer.
+There are a variety of approaches on the usage of pillar and templates as seen in the `saltstack-formulas <https://github.com/saltstack-formulas>`_' repositories. `Some <https://github.com/saltstack-formulas/nginx-formula/pull/18>`_ `developments <https://github.com/saltstack-formulas/php-formula/pull/14>`_ stress the initial purpose of pillar data into a storage for most of the possible variables for a determined system configuration. This, in my opinion, is shifting too much load from the original template files approach. Adding up some `non-trivial Jinja <https://github.com/saltstack-formulas/nginx-formula/blob/f74254c07e188bd448eaf1c5f9c802d78c4c005e/nginx/files/default/nginx.conf>`_ code as essential part of composing the state file definitely makes SaltStack state files (hence formulas) more difficult to read. The extreme of this approach is that we could end up with a new render mechanism, implemented in Jinja, storing everything needed in pillar data to compose configurations. Additionally, we are establishing a strong dependency with the Jinja renderer.
 In opposition to the *put the code in file_roots and the data in pillars* approach, there is the *pillar as a store for a set of key-values* approach. A full-blown configuration file abstracted in pillar and jinja is complicated to develop, understand and maintain. I think a better and simpler approach is to keep a configuration file templated using just a basic (non-extensive but extensible) set of pillar values.
@@ -325,6 +325,7 @@ We can simplify the ``conf.sls`` with the new ``files_switch`` macro to use in t
 * This uses ``config.get``, searching for ``ntp:tofs:source_files:Configure NTP`` to determine the list of template files to use.
+* If this returns a result, the default of ``['/etc/ntp.conf.jinja']`` will be appended to it.
 * If this does not yield any results, the default of ``['/etc/ntp.conf.jinja']`` will be used.
 In ``libtofs.jinja``, we define this new macro ``files_switch``.
@@ -426,7 +427,6 @@ The list of ``source_files`` can be given:
     tofs:
       source_files:
         Configure NTP:
-           - '/etc/ntp.conf.jinja'
           - '/etc/ntp.conf_alt.jinja'
 Resulting in:
@@ -434,10 +434,85 @@ Resulting in:
 .. code-block:: sls
         - source:
-           - salt://ntp/files/theminion/etc/ntp.conf.jinja
           - salt://ntp/files/theminion/etc/ntp.conf_alt.jinja
-           - salt://ntp/files/Debian/etc/ntp.conf.jinja
+           - salt://ntp/files/theminion/etc/ntp.conf.jinja
           - salt://ntp/files/Debian/etc/ntp.conf_alt.jinja
-           - salt://ntp/files/default/etc/ntp.conf.jinja
+           - salt://ntp/files/Debian/etc/ntp.conf.jinja
           - salt://ntp/files/default/etc/ntp.conf_alt.jinja
+           - salt://ntp/files/default/etc/ntp.conf.jinja
+Note: This does *not* override the default value.
+Rather, the value from the pillar/config is prepended to the default.
+Using sub-directories for ``components``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+If your formula is composed of several components, you may prefer to provides files under sub-directories, like in the `systemd-formula <https://github.com/saltstack-formulas/systemd-formula>`_.
+.. code-block::
+   /srv/saltstack/systemd-formula/
+     systemd/
+       init.sls
+       libtofs.jinja
+       map.jinja
+       networkd/
+         init.sls
+         files/
+           default/
+             network/
+               99-default.link
+       resolved/
+         init.sls
+         files/
+           default/
+             resolved.conf
+       timesyncd/
+         init.sls
+         files/
+           Arch/
+             resolved.conf
+           Debian/
+             resolved.conf
+           default/
+             resolved.conf
+           Ubuntu/
+             resolved.conf
+For example, the following ``formula.component.config`` SLS:
+.. code-block:: sls
+   {%- from "formula/libtofs.jinja" import files_switch with context %}
+   formula configuration file:
+     file.managed:
+       - name: /etc/formula.conf
+       - user: root
+       - group: root
+       - mode: 644
+       - template: jinja
+       - source: {{ files_switch(['formula.conf'],
+                                 lookup='formula',
+                                 use_subpath=True
+                    )
+                 }}
+will be rendered on a ``Debian`` minion named ``salt-formula.ci.local`` as:
+.. code-block:: sls
+   formula configuration file:
+     file.managed:
+       - name: /etc/formula.conf
+       - user: root
+       - group: root
+       - mode: 644
+       - template: jinja
+       - source:
+         - salt://formula/component/files/salt-formula.ci.local/formula.conf
+         - salt://formula/component/files/Debian/formula.conf
+         - salt://formula/component/files/default/formula.conf
+         - salt://formula/files/salt-formula.ci.local/formula.conf
+         - salt://formula/files/Debian/formula.conf
+         - salt://formula/files/default/formula.conf
--- a/kitchen.yml
+++ b/kitchen.yml
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
 ---
 driver:
  name: docker
@@ -11,17 +13,22 @@ platforms:
  - name: debian-9
  - name: ubuntu-18.04
  - name: centos-7
-  - name: fedora-29
+  - name: fedora-30
-  - name: opensuse-42.3
+  - name: opensuse-15.0
    driver:
+      image: opensuse/leap:15.0
      run_command: /usr/lib/systemd/systemd
      provision_command:
        - systemctl enable sshd.service
+    # Workaround to avoid intermittent failures on `opensuse-leap-15`:
+    # => SCP did not finish successfully (255):  (Net::SCP::Error)
+    transport:
+      max_ssh_sessions: 1
  # Previous distros
  - name: debian-8
  - name: ubuntu-16.04
-  - name: fedora-28
+  - name: fedora-29
  # centos-6 guest fails on Debian hosts due to vsyscall issues, see
  # https://hub.docker.com/_/centos, "A note about vsyscall"
  - name: centos-6
@@ -43,55 +50,58 @@ provisioner:
        - salt.pkgrepo
        - salt.master
        - salt.minion
-  pillars:
-    top.sls:
-      base:
-        '*':
-          - salt
-    salt.sls:
-      salt:
-        master:
-          fileserver_backend:
-            - rootfs
-          file_roots:
-            base:
-              - /srv/salt
-          pillar_roots:
-            base:
-              - /srv/pillar
-        minion:
-          master: localhost
-          fileserver_backend:
-            - rootfs
-          file_roots:
-            base:
-              - /srv/salt
-          pillar_roots:
-            base:
-              - /srv/pillar
 verifier:
+  # https://www.inspec.io/
  name: inspec
  sudo: true
+  # cli, documentation, html, progress, json, json-min, json-rspec, junit
  reporter:
    - cli
 suites:
+  # Latest distros, salt develop, python3
+  # These distros have py3 packages available in salt's repo
+  - name: develop-py3
+    includes:
+      - debian-9
+      - ubuntu-18.04
+      - opensuse-15.0
+    provisioner:
+      salt_bootstrap_options: -X -x python3 -d git %s
+      salt_version: 'develop'
+      pillars:
+        top.sls:
+          base:
+            '*':
+              - salt
+              - develop-py3
+      pillars_from_files:
+        salt.sls: test/salt/pillar/salt.sls
+        develop-py3.sls: test/salt/pillar/develop-py3.sls
+    verifier:
+      inspec_tests:
+        - path: test/integration/develop
  # Latest distros, latest salt, python3
  # These distros have py3 packages available in salt's repo
  - name: v2019-2-py3
    includes:
      - debian-9
      - ubuntu-18.04
+      - opensuse-15.0
    provisioner:
      salt_bootstrap_options: -X -x python3 -d git %s
      salt_version: '2019.2'
      pillars:
-        salt.sls:
+        top.sls:
-          salt:
+          base:
-            release: '2019.2'
+            '*':
-            py_ver: 'py3'
+              - salt
+              - v2019-2-py3
+      pillars_from_files:
+        salt.sls: test/salt/pillar/salt.sls
+        v2019-2-py3.sls: test/salt/pillar/v2019-2-py3.sls
    verifier:
      inspec_tests:
        - path: test/integration/2019-2
@@ -101,15 +111,18 @@ suites:
  - name: v2019-2-py2
    includes:
      - centos-7
-      - fedora-29
+      - fedora-30
    provisioner:
      salt_version: '2019.2'
      pillars:
-        salt.sls:
+        top.sls:
-          salt:
+          base:
-            release: '2019.2'
+            '*':
-            py_ver: 'py2'
+              - salt
+              - v2019-2-py2
+      pillars_from_files:
+        salt.sls: test/salt/pillar/salt.sls
+        v2019-2-py2.sls: test/salt/pillar/v2019-2-py2.sls
    verifier:
      inspec_tests:
        - path: test/integration/2019-2
@@ -119,53 +132,42 @@ suites:
    includes:
      - debian-8
      - ubuntu-16.04
-      - opensuse-42.3
+      - centos-7
    provisioner:
      # We require an old version of salt in the provisioner or,
      # the salt formula fails to downgrade to the desired version to test
      salt_version: '2018.3'
      pillars:
-        salt.sls:
+        top.sls:
-          salt:
+          base:
-            release: '2018.3'
+            '*':
-            py_ver: 'py2'
+              - salt
-    verifier:
+              - v2018-3-py2
-      inspec_tests:
+      pillars_from_files:
-        - path: test/integration/2018-3
+        salt.sls: test/salt/pillar/salt.sls
+        v2018-3-py2.sls: test/salt/pillar/v2018-3-py2.sls
-  # centos-6 ships with python2.6, so it requires extra bootstrapping parameters
-  # to install python2.7
-  - name: v2018-3-py2-bootstrap
-    includes:
-      - centos-6
-    provisioner:
-      salt_bootstrap_options: -X -d stable %s
-      salt_version: '2018.3'
-      pillars:
-        salt.sls:
-          salt:
-            release: '2018.3'
-            py_ver: 'py2'
    verifier:
      inspec_tests:
        - path: test/integration/2018-3
-  # To tests fedora 28 & salt v2018.2, we need to force the package version
+  # To test fedora 29 & salt v2018.3, we need to force the package version
  # otherwise the image, which includes the 'updates' repo, will install 2019.2
  - name: v2018-3-py2-forced-version
    includes:
-      - fedora-28
+      - fedora-29
    provisioner:
      # We require an old version of salt in the provisioner or,
      # the salt formula fails to downgrade to the desired version to test
      salt_version: '2018.3'
      pillars:
-        salt.sls:
+        top.sls:
-          salt:
+          base:
-            release: '2018.3'
+            '*':
-            py_ver: 'py2'
+              - salt
-            version: '2018.3.0-1.fc28'
+              - v2018-3-py2-forced-version
+      pillars_from_files:
+        salt.sls: test/salt/pillar/salt.sls
+        v2018-3-py2-forced-version.sls: test/salt/pillar/v2018-3-py2-forced-version.sls
    verifier:
      inspec_tests:
        - path: test/integration/2018-3
@@ -180,10 +182,14 @@ suites:
      # the salt formula fails to downgrade to the desired version to test
      salt_version: '2017.7'
      pillars:
-        salt.sls:
+        top.sls:
-          salt:
+          base:
-            release: '2017.7'
+            '*':
-            py_ver: 'py2'
+              - salt
+              - v2017-7-py2
+      pillars_from_files:
+        salt.sls: test/salt/pillar/salt.sls
+        v2017-7-py2.sls: test/salt/pillar/v2017-7-py2.sls
    verifier:
      inspec_tests:
        - path: test/integration/2017-7
@@ -198,11 +204,14 @@ suites:
      salt_bootstrap_options: -X -d stable %s
      salt_version: '2017.7'
      pillars:
-        salt.sls:
+        top.sls:
-          salt:
+          base:
-            release: '2017.7'
+            '*':
-            py_ver: 'py2'
+              - salt
+              - v2017-7-py2
+      pillars_from_files:
+        salt.sls: test/salt/pillar/salt.sls
+        v2017-7-py2.sls: test/salt/pillar/v2017-7-py2.sls
    verifier:
      inspec_tests:
        - path: test/integration/2017-7
--- a/pillar.example
+++ b/pillar.example
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+---
 salt:
  # Set this to true to clean any non-salt-formula managed files out of
  # /etc/salt/{master,minion}.d ... You really don't want to do this on 2015.2
  # and up as it'll wipe out important files that Salt relies on.
-  clean_config_d_dir: False
+  clean_config_d_dir: false
  # This state will remove "/etc/salt/minion" when you set this to true.
-  minion_remove_config: True
+  minion_remove_config: true
  # This state will remove "/etc/salt/master" when you set this to true.
-  master_remove_config: True
+  master_remove_config: true
  # Set this to 'py3' to install the Python 3 packages.
  # If this is not set, the Python 2 packages will be installed by default.
  py_ver: 'py3'
-  # Set this to False to not have the formula install packages (in the case you
+  # Set this to false to not have the formula install packages (in the case you
  # install Salt via git/pip/etc.)
-  install_packages: True
+  install_packages: true
-  # Optional: set salt version (if install_packages is set to True)
+  # Optional: set salt version (if install_packages is set to true)
  version: 2017.7.2-1.el7
  # to overwrite map.jinja salt packages
@@ -37,14 +40,18 @@ salt:
  release: '2018.3'
  # MacOS has no package management.
-  # Instead, we use file.managed to download an appropriate .pkg file and macpackage.installed to install it
+  # Instead, we use file.managed to download an appropriate .pkg file and
-  # 'version', if set (see above), will be used to check the .pkg version to determine if it should be installed
+  # macpackage.installed to install it 'version', if set (see above), will be
+  # used to check the .pkg version to determine if it should be installed
  #
-  # NOTE: if 'version' is not set version comparison will not occur and the .pkg WILL NOT be installed if a salt
+  # NOTE: if 'version' is not set version comparison will not occur and the
-  # .pkg is already installed
+  # .pkg WILL NOT be installed if a salt .pkg is already installed
-  # NOTE: salt_minion_pkg_hash, if set, will be passed into file.managed's source_hash, use URL or hash string
+  # NOTE: salt_minion_pkg_hash, if set, will be passed into file.managed's
+  # source_hash, use URL or hash string
+  # yamllint disable rule:line-length
  salt_minion_pkg_source: 'https://repo.saltstack.com/osx/salt-2017.7.4-py3-x86_64.pkg'
  salt_minion_pkg_hash: 'https://repo.saltstack.com/osx/salt-2017.7.4-py3-x86_64.pkg.md5'
+  # yamllint enable rule:line-length
  # tofs:
  #   The files_switch key serves as a selector for alternative
@@ -71,7 +78,7 @@ salt:
  #       - 'alt_minion.d'
  # salt master config
-  master_config_use_TOFS: True
+  master_config_use_TOFS: true
  master:
    fileserver_backend:
      - git
@@ -98,8 +105,9 @@ salt:
      port: 8000
      ssl_crt: /etc/pki/api/certs/server.crt
      ssl_key: /etc/pki/api/certs/server.key
-      debug: False
+      debug: false
-      disable_ssl: False
+      disable_ssl: false
+      # yamllint disable-line rule:line-length
    # for profile configuration as https://docs.saltstack.com/en/latest/topics/tutorials/lxc.html#tutorial-lxc-profiles
    lxc.container_profile:
      debian:
@@ -118,10 +126,10 @@ salt:
          type: veth
          flags: up
    ## for external auth - LDAP
-      # filter to use for Active Directory LDAP
+    ## filter to use for Active Directory LDAP
-    auth.ldap.filter: {% raw %}'sAMAccountName={{username}}'{% endraw %}
+    # auth.ldap.filter: {% raw %}'sAMAccountName={{username}}'{% endraw %}
-      # filter to use for Most other LDAP servers
+    ## filter to use for Most other LDAP servers
-    auth.ldap.filter: {% raw %}'uid={{ username }}'{% endraw %}
+    # auth.ldap.filter: {% raw %}'uid={{ username }}'{% endraw %}
    # Define winrepo provider, by default support order is pygit2, gitpython
    # Set to gitpython for Debian & Ubuntu to get around saltstack/salt#35993
@@ -132,7 +140,7 @@ salt:
    engines:
      - slack:
          token: xoxp-XXXXX-XXXXXXX   # use Slack's legacy API token
-          control: True
+          control: true
          valid_users:
            - someuser
            - otheruser
@@ -152,7 +160,7 @@ salt:
          - /srv/salt/reactors/deploy.sls
  # salt minion config:
-  minion_config_use_TOFS: True
+  minion_config_use_TOFS: true
  minion:
    # single master setup
@@ -176,7 +184,7 @@ salt:
      base:
        - /srv/pillar
    module_config:
-      test: True
+      test: true
      test.foo: foo
      test.bar:
        - baz
@@ -209,7 +217,7 @@ salt:
    engines:
      - slack:
          token: xoxp-XXXXX-XXXXXXX   # use Slack's legacy API token
-          control: True
+          control: true
          valid_users:
            - someuser
            - otheruser
@@ -248,14 +256,14 @@ salt:
      hosts:
        - example.elasticsearch.host:9200
        - example.elasticsearch.host2:9200
-      index_date: True
+      index_date: true
      index: salt
      number_of_shards: 5
      number_of_replicas: 2
-      debug_returner_payload: True
+      debug_returner_payload: true
-      states_count: True
+      states_count: true
-      states_order_output: True
+      states_order_output: true
-      states_single_index: True
+      states_single_index: true
      functions_blacklist:
        - test.ping
        - saltutil.find_job
@@ -307,6 +315,7 @@ salt:
    aws_key: AWSKEYIJSHJAIJS6JSH
    aws_secret: AWSSECRETYkkDY1iQf9zRtl9+pW+Nm+aZY95
    gce_project: test
+    # yamllint disable-line rule:line-length
    gce_service_account_email_address: 867543072364-orl4h2tpp8jcn1tr9ipj@developer.gserviceaccount.com
    rsos_user: afeawofghob
    rsos_tenant: tenant_id_number
@@ -322,12 +331,13 @@ salt:
    prod1:
      host: host.example.com
      user: ubuntu
-      sudo: True
+      sudo: true
      priv: /etc/salt/ssh_keys/sshkey.pem
  gitfs:
    keys:
      global:
-        # key and pub end up being the extension used on the key file. values other than key and pub are possible
+        # key and pub end up being the extension used on the key file
+        # values other than key and pub are possible
        key: |
          -----BEGIN RSA PRIVATE KEY-----
          ...........
@@ -364,8 +374,8 @@ salt_formulas:
      baseurl: https://github.com/saltstack-formulas
      # Directory where Git repositories are downloaded
      basedir: /srv/formulas
-      # Update the git repository to the latest version (False by default)
+      # Update the git repository to the latest version (false by default)
-      update: False
+      update: false
      # Options passed directly to the git.latest state
      options:
        rev: master
@@ -373,7 +383,7 @@ salt_formulas:
        identity: /path/to/.ssh/id_rsa_github_username
    dev:
      basedir: /srv/formulas/dev
-      update: True
+      update: true
      options:
        rev: develop
    # Alternatively, a single directory with multiple branches can be used
@@ -394,21 +404,21 @@ salt_formulas:
        rev: staging
    upstream:
      baseurl: git@github.com:saltstack-formulas
-      update: True
+      update: true
      options:
        branch: upstream
        remote: upstream
  # Options of the file.directory state that creates the directory where
  # the git repositories of the formulas are stored
  basedir_opts:
-    makedirs: True
+    makedirs: true
    user: root
    group: root
    mode: 755
  # Explicitly checkout the original branch for repos after the
-  # git.latest states have been processed (False by default)
+  # git.latest states have been processed (false by default)
  # Enable if using the alternative method (single directory, multiple branches)
-  checkout_orig_branch: True
+  checkout_orig_branch: true
  # List of formulas to enable in each environment
  list:
    base:
@@ -430,4 +440,3 @@ salt_formulas:
      - salt-formula
      - postfix-formula
      - openssh-formula
--- a/salt/defaults.yaml
+++ b/salt/defaults.yaml
 # -*- coding: utf-8 -*-
 # vim: ft=yaml
+---
 salt:
  version: ''
  rootuser: root
  rootgroup: root
-  install_packages: True
+  install_packages: true
-  use_pip: False
+  use_pip: false
-  clean_config_d_dir: True
+  clean_config_d_dir: true
-  restart_via_at: False
+  restart_via_at: false
  config_path: /etc/salt
-  minion_remove_config: False
+  minion_remove_config: false
-  master_remove_config: False
+  master_remove_config: false
-  minion_config_use_TOFS: False
+  minion_config_use_TOFS: false
-  master_config_use_TOFS: False
+  master_config_use_TOFS: false
  minion_service: salt-minion
  master_service: salt-master
@@ -36,22 +37,22 @@ salt:
  gitfs:
    dulwich:
-      install_from_source: True
+      install_from_source: true
    pygit2:
-      install_from_source: True
+      install_from_source: true
      version: 0.23.0
      git:
        # if not false, should be state name
-        require_state: False
+        require_state: false
        install_from_package: git
      libgit2:
        version: 0.23.0
-        install_from_source: True
+        install_from_source: true
        build_parent_dir: /usr/src/
        # hash necessary until github issue #9272 is addressed
        download_hash: 683d1164e361e2a0a8d52652840e2340
    gitpython:
-      install_from_source: False
+      install_from_source: false
  cloud:
    template_sources:
@@ -60,12 +61,12 @@ salt:
      maps: salt://salt/files/cloud.maps.d
 salt_formulas:
-  checkout_orig_branch: False
+  checkout_orig_branch: false
  git_opts:
    default:
      baseurl: https://github.com/saltstack-formulas
      basedir: /srv/formulas
-      update: False
+      update: false
      options:
        rev: master
        output_loglevel: 'quiet'
--- a/salt/libtofs.jinja
+++ b/salt/libtofs.jinja
@@ -2,7 +2,7 @@
                       lookup=None,
                       default_files_switch=['id', 'os_family'],
                       indent_width=6,
-                       v1_path_prefix='') %}
+                       use_subpath=False) %}
  {#-
    Returns a valid value for the "source" parameter of a "file.managed"
    state function. This makes easier the usage of the Template Override and
@@ -10,15 +10,15 @@
    Params:
      * source_files: ordered list of files to look for
-      * lookup: key under '<tplroot>:tofs:source_files' to override
+      * lookup: key under '<tplroot>:tofs:source_files' to prepend to the
        list of source files
      * default_files_switch: if there's no config (e.g. pillar)
        '<tplroot>:tofs:files_switch' this is the ordered list of grains to
        use as selector switch of the directories under
        "<path_prefix>/files"
      * indent_witdh: indentation of the result value to conform to YAML
-      * v1_path_prefix: (deprecated) only used for injecting a path prefix into
+      * use_subpath: defaults to `False` but if set, lookup the source file
-        the source, to support older TOFS configs
+        recursively from the current state directory up to `tplroot`
    Example (based on a `tplroot` of `xxx`):
@@ -55,20 +55,24 @@
      tplroot ~ ':tofs:files_switch',
      default_files_switch
  ) %}
-  {#- Lookup source_files (v2), files (v1), or fallback to source_files parameter #}
+  {#- Lookup source_files (v2), files (v1), or fallback to an empty list #}
  {%- set src_files = salt['config.get'](
      tplroot ~ ':tofs:source_files:' ~ lookup,
-      salt['config.get'](
+      salt['config.get'](tplroot ~ ':tofs:files:' ~ lookup, [])
-          tplroot ~ ':tofs:files:' ~ lookup,
-          source_files
-      )
  ) %}
+  {#- Append the default source_files #}
+  {%- set src_files = src_files + source_files %}
  {#- Only add to [''] when supporting older TOFS implementations #}
  {%- set path_prefix_exts = [''] %}
-  {%- if v1_path_prefix != '' %}
+  {%- if use_subpath and tplroot != tpldir %}
-    {%- do path_prefix_exts.append(v1_path_prefix) %}
+    {#- Walk directory tree to find {{ files_dir }} #}
+    {%- set subpath_parts = tpldir.lstrip(tplroot).lstrip('/').split('/') %}
+    {%- for path in subpath_parts %}
+      {%- set subpath = subpath_parts[0:loop.index] | join('/') %}
+      {%- do path_prefix_exts.append('/' ~ subpath) %}
+    {%- endfor %}
  {%- endif %}
-  {%- for path_prefix_ext in path_prefix_exts %}
+  {%- for path_prefix_ext in path_prefix_exts|reverse %}
    {%- set path_prefix_inc_ext = path_prefix ~ path_prefix_ext %}
    {#- For older TOFS implementation, use `files_switch` from the config #}
    {#- Use the default, new method otherwise #}
@@ -83,10 +87,16 @@
    {%- for fs in fsl %}
      {%- for src_file in src_files %}
        {%- if fs %}
-          {%- set fs_dir = salt['config.get'](fs, fs) %}
+          {%- set fs_dirs = salt['config.get'](fs, fs) %}
        {%- else %}
-          {%- set fs_dir = salt['config.get'](tplroot ~ ':tofs:dirs:default', 'default') %}
+          {%- set fs_dirs = salt['config.get'](tplroot ~ ':tofs:dirs:default', 'default') %}
+        {%- endif %}
+        {#- Force the `config.get` lookup result as a list where necessary #}
+        {#- since we need to also handle grains that are lists #}
+        {%- if fs_dirs is string %}
+          {%- set fs_dirs = [fs_dirs] %}
        {%- endif %}
+        {%- for fs_dir in fs_dirs %}
          {%- set url = [
              '- salt:/',
              path_prefix_inc_ext.strip('/'),
@@ -98,4 +108,5 @@
        {%- endfor %}
      {%- endfor %}
    {%- endfor %}
+  {%- endfor %}
 {%- endmacro %}
--- a/salt/osfamilymap.yaml
+++ b/salt/osfamilymap.yaml
 # -*- coding: utf-8 -*-
 # vim: ft=yaml
+---
 {% import_yaml "salt/ospyvermap.yaml" as ospyvermap %}
 {% set ospyver = salt['grains.filter_by'](ospyvermap, grain='os_family') or {} %}
@@ -27,13 +28,13 @@ Debian:
  pyinotify: python-pyinotify
  gitfs:
    pygit2:
-      install_from_source: True
+      install_from_source: true
      version: 0.22.1
      git:
-        require_state: False
+        require_state: false
        install_from_package: git
      libgit2:
-        install_from_source: False
+        install_from_source: false
 RedHat:
  pkgrepo: 'https://repo.saltstack.com/{{ py_ver_repr }}/redhat/$releasever/$basearch/{{ salt_release }}'
@@ -42,11 +43,11 @@ RedHat:
  python_git: GitPython
  gitfs:
    gitpython:
-      install_from_source: False
+      install_from_source: false
    pygit2:
-      install_from_source: False
+      install_from_source: false
      git:
-        require_state: False
+        require_state: false
        install_from_package: git
  master:
    gitfs_provider: pygit2
@@ -58,9 +59,9 @@ Suse:
  pyinotify: python-pyinotify
  gitfs:
    pygit2:
-      install_from_source: False
+      install_from_source: false
      git:
-        require_state: False
+        require_state: false
        install_from_package: git
  master:
    gitfs_provider: pygit2

--- a/salt/osmap.yaml
+++ b/salt/osmap.yaml
 # -*- coding: utf-8 -*-
 # vim: ft=yaml
+---
 {% import_yaml "salt/ospyvermap.yaml" as ospyvermap %}
 {% set ospyver = salt['grains.filter_by'](ospyvermap, grain='os_family') or {} %}
@@ -24,9 +25,9 @@ Ubuntu:
  pygit2: python-pygit2
  gitfs:
    pygit2:
-      install_from_source: False
+      install_from_source: false
      git:
-        require_state: False
+        require_state: false
        install_from_package: Null
 Raspbian:
@@ -46,7 +47,7 @@ SmartOS:
  python_dulwich: 'py27-dulwich'
  gitfs:
    dulwich:
-      install_from_source: False
+      install_from_source: false
  config_path: /opt/local/etc/salt
  master:
    gitfs_provider: dulwich
--- a/salt/ospyvermap.yaml
+++ b/salt/ospyvermap.yaml
 # -*- coding: utf-8 -*-
 # vim: ft=yaml
+---
 Debian:
  py2: 'apt'
  py3: 'py3'

--- a/salt/pkgrepo/suse/install.sls
+++ b/salt/pkgrepo/suse/install.sls
@@ -5,7 +5,7 @@
 salt-pkgrepo-install-saltstack-suse:
  pkgrepo.managed:
    - name: systemsmanagement_saltstack_products
-    - humanname: SaltStack repo for Opensuse 42.3
+    - humanname: SaltStack repo for Opensuse 15.x
    - baseurl: {{ salt_settings.pkgrepo }}
    - enabled: 1
    - gpgcheck: 1

--- a/test/integration/2018-3/pkgs_spec.rb
+++ b/test/integration/2018-3/pkgs_spec.rb
 case os[:name]
 when 'centos'
-  version = '2018.3.4-1.el6'
+  version = '2018.3.4-1.el7'
 when 'debian', 'ubuntu'
  version = '2018.3.4+ds-1'
-when 'opensuse'
-  version = '2018.3.0-42.1'
 when 'fedora'
-  version = '2018.3.0-1.fc28'
+  version = '2018.3.2-5.fc29'
 end
 control 'salt packages' do

--- a/test/integration/2019-2/pkgs_spec.rb
+++ b/test/integration/2019-2/pkgs_spec.rb
@@ -2,7 +2,9 @@ case os[:name]
 when 'centos'
  version = '2019.2.0-1.el7'
 when 'fedora'
-  version = '2019.2.0-1.fc29'
+  version = '2019.2.0-1.fc30'
+when 'opensuse'
+  version = '2019.2.0-lp150.3.33.1'
 when 'debian', 'ubuntu'
  version = '2019.2.0+ds-1'
 end

--- a/test/integration/develop/pkgs_spec.rb
+++ b/test/integration/develop/pkgs_spec.rb
+control 'salt packages' do
+  title 'should be installed'
+  %w(
+    salt-master
+    salt-minion
+  ).each do |p|
+    describe package(p) do
+      it { should be_installed }
+    end
+  end
+end
--- a/test/integration/develop/service_spec.rb
+++ b/test/integration/develop/service_spec.rb
+control 'salt services' do
+  title 'should be running'
+  %w(
+    salt-master
+    salt-minion
+  ).each do |p|
+    describe service(p) do
+      it { should be_enabled }
+      it { should be_running }
+    end
+  end
+end