From 0ba4954a4ba4ca91c188354d46cba16e7c27d23d Mon Sep 17 00:00:00 2001 From: Justin Findlay Date: Tue, 13 Feb 2018 23:46:55 -0800 Subject: [PATCH] salt.pillar.file_tree revise module documentation --- salt/pillar/file_tree.py | 338 ++++++++++++++++++++++----------------- 1 file changed, 188 insertions(+), 150 deletions(-) diff --git a/salt/pillar/file_tree.py b/salt/pillar/file_tree.py index ede20f798b..97187bedde 100644 --- a/salt/pillar/file_tree.py +++ b/salt/pillar/file_tree.py @@ -1,175 +1,97 @@ # -*- coding: utf-8 -*- ''' +The ``file_tree`` external pillar allows values from all files in a directory +tree to be imported as Pillar data. -``File_tree`` is an external pillar that allows -values from all files in a directory tree to be imported as Pillar data. +.. note:: -Note this is an external pillar, and is subject to the rules and constraints -governing external pillars detailed here: :ref:`external-pillars`. + This is an external pillar and is subject to the :ref:`rules and + constraints ` governing external pillars. .. versionadded:: 2015.5.0 -Example Configuration ---------------------- +In this pillar, data is organized by either Minion ID or Nodegroup name. To +setup pillar data for a specific Minion, place it in +``/hosts/``. To setup pillar data for an entire +Nodegroup, place it in ``/nodegroups/`` where +```` is the Nodegroup's name. + +Example ``file_tree`` Pillar +============================ + +Master Configuration +-------------------- .. code-block:: yaml ext_pillar: - file_tree: - root_dir: /path/to/root/directory + root_dir: /srv/ext_pillar follow_dir_links: False keep_newline: True -The ``root_dir`` parameter is required and points to the directory where files -for each host are stored. The ``follow_dir_links`` parameter is optional and -defaults to False. If ``follow_dir_links`` is set to True, this external pillar -will follow symbolic links to other directories. + node_groups: + internal_servers: 'L@bob,stuart,kevin' -.. warning:: - Be careful when using ``follow_dir_links``, as a recursive symlink chain - will result in unexpected results. +Pillar Configuration +-------------------- -If ``keep_newline`` is set to ``True``, then the pillar values for files ending -in newlines will keep that newline. The default behavior is to remove the -end-of-file newline. ``keep_newline`` should be turned on if the pillar data is -intended to be used to deploy a file using ``contents_pillar`` with a -:py:func:`file.managed ` state. +.. code-block:: bash -.. versionchanged:: 2015.8.4 - The ``raw_data`` parameter has been renamed to ``keep_newline``. In earlier - releases, ``raw_data`` must be used. Also, this parameter can now be a list - of globs, allowing for more granular control over which pillar values keep - their end-of-file newline. The globs match paths relative to the - directories named for minion IDs and nodegroups underneath the ``root_dir`` - (see the layout examples in the below sections). + (salt-master) # tree /srv/ext_pillar + /srv/ext_pillar/ + |-- hosts + | |-- bob + | | |-- apache + | | | `-- config.d + | | | |-- 00_important.conf + | | | `-- 20_bob_extra.conf + | | `-- corporate_app + | | `-- settings + | | `-- bob_settings.cfg + | `-- kevin + | |-- apache + | | `-- config.d + | | `-- 00_important.conf + | `-- corporate_app + | `-- settings + | `-- kevin_settings.cfg + `-- nodegroups + `-- internal_servers + `-- corporate_app + `-- settings + `-- common_settings.cfg - .. code-block:: yaml +Verify Pillar Data +------------------ - ext_pillar: - - file_tree: - root_dir: /path/to/root/directory - keep_newline: - - files/testdir/* +.. code-block:: bash + + (salt-master) # salt bob pillar.items + bob: + ---------- + apache: + ---------- + config.d: + ---------- + 00_important.conf: + + 20_bob_extra.conf: + + corporate_app: + ---------- + settings: + ---------- + common_settings: + // This is the main settings file for the corporate + // internal web app + main_setting: probably + bob_settings: + role: bob .. note:: - In earlier releases, this documentation incorrectly stated that binary - files would not affected by the ``keep_newline`` configuration. However, - this module does not actually distinguish between binary and text files. -.. versionchanged:: 2017.7.0 - Templating/rendering has been added. You can now specify a default render - pipeline and a black- and whitelist of (dis)allowed renderers. - - ``template`` must be set to ``True`` for templating to happen. - - .. code-block:: yaml - - ext_pillar: - - file_tree: - root_dir: /path/to/root/directory - render_default: jinja|yaml - renderer_blacklist: - - gpg - renderer_whitelist: - - jinja - - yaml - template: True - -Assigning Pillar Data to Individual Hosts ------------------------------------------ - -To configure pillar data for each host, this external pillar will recursively -iterate over ``root_dir``/hosts/``id`` (where ``id`` is a minion ID), and -compile pillar data with each subdirectory as a dictionary key and each file -as a value. - -For example, the following ``root_dir`` tree: - -.. code-block:: text - - ./hosts/ - ./hosts/test-host/ - ./hosts/test-host/files/ - ./hosts/test-host/files/testdir/ - ./hosts/test-host/files/testdir/file1.txt - ./hosts/test-host/files/testdir/file2.txt - ./hosts/test-host/files/another-testdir/ - ./hosts/test-host/files/another-testdir/symlink-to-file1.txt - -will result in the following pillar tree for minion with ID ``test-host``: - -.. code-block:: text - - test-host: - ---------- - files: - ---------- - another-testdir: - ---------- - symlink-to-file1.txt: - Contents of file #1. - - testdir: - ---------- - file1.txt: - Contents of file #1. - - file2.txt: - Contents of file #2. - -.. note:: - Subdirectories underneath ``root_dir``/hosts/``id`` become nested - dictionaries, as shown above. - - -Assigning Pillar Data to Entire Nodegroups ------------------------------------------- - -To assign Pillar data to all minions in a given nodegroup, this external pillar -recursively iterates over ``root_dir``/nodegroups/``nodegroup`` (where -``nodegroup`` is the name of a nodegroup), and like for individual hosts, -compiles pillar data with each subdirectory as a dictionary key and each file -as a value. - -.. important:: - If the same Pillar key is set for a minion both by nodegroup and by - individual host, then the value set for the individual host will take - precedence. - -For example, the following ``root_dir`` tree: - -.. code-block:: text - - ./nodegroups/ - ./nodegroups/test-group/ - ./nodegroups/test-group/files/ - ./nodegroups/test-group/files/testdir/ - ./nodegroups/test-group/files/testdir/file1.txt - ./nodegroups/test-group/files/testdir/file2.txt - ./nodegroups/test-group/files/another-testdir/ - ./nodegroups/test-group/files/another-testdir/symlink-to-file1.txt - -will result in the following pillar data for minions in the node group -``test-group``: - -.. code-block:: text - - test-host: - ---------- - files: - ---------- - another-testdir: - ---------- - symlink-to-file1.txt: - Contents of file #1. - - testdir: - ---------- - file1.txt: - Contents of file #1. - - file2.txt: - Contents of file #2. + The leaf data in the example shown is the contents of the pillar files. ''' from __future__ import absolute_import @@ -302,7 +224,123 @@ def ext_pillar(minion_id, renderer_whitelist=None, template=False): ''' - Compile pillar data for the specified minion ID + Compile pillar data from the given ``root_dir`` specific to Nodegroup names + and Minion IDs. + + If a Minion's ID is not found at ``/host/`` or if it + is not included in any Nodegroups named at + ``/nodegroups/``, no pillar data provided by this + pillar module will be available for that Minion. + + .. versionchanged:: 2017.7.0 + Templating/rendering has been added. You can now specify a default + render pipeline and a black- and whitelist of (dis)allowed renderers. + + :param:`template` must be set to ``True`` for templating to happen. + + .. code-block:: yaml + + ext_pillar: + - file_tree: + root_dir: /path/to/root/directory + render_default: jinja|yaml + renderer_blacklist: + - gpg + renderer_whitelist: + - jinja + - yaml + template: True + + :param minion_id: + The ID of the Minion whose pillar data is to be collected + + :param pillar: + Unused by the ``file_tree`` pillar module + + :param root_dir: + Filesystem directory used as the root for pillar data (e.g. + ``/srv/ext_pillar``) + + :param follow_dir_links: + Follow symbolic links to directories while collecting pillar files. + Defaults to ``False``. + + .. warning:: + + Care should be exercised when enabling this option as it will + follow links that point outside of :param:`root_dir`. + + .. warning:: + + Symbolic links that lead to infinite recursion are not filtered. + + :param debug: + Enable debug information at log level ``debug``. Defaults to + ``False``. This option may be useful to help debug errors when setting + up the ``file_tree`` pillar module. + + :param keep_newline: + Preserve the end-of-file newline in files. Defaults to ``False``. + This option may either be a boolean or a list of file globs (as defined + by the `Python fnmatch package + `_) for which end-of-file + newlines are to be kept. + + ``keep_newline`` should be turned on if the pillar data is intended to + be used to deploy a file using ``contents_pillar`` with a + :py:func:`file.managed ` state. + + .. versionchanged:: 2015.8.4 + The ``raw_data`` parameter has been renamed to ``keep_newline``. In + earlier releases, ``raw_data`` must be used. Also, this parameter + can now be a list of globs, allowing for more granular control over + which pillar values keep their end-of-file newline. The globs match + paths relative to the directories named for Minion IDs and + Nodegroup namess underneath the :param:`root_dir`. + + .. code-block:: yaml + + ext_pillar: + - file_tree: + root_dir: /srv/ext_pillar + keep_newline: + - apache/config.d/* + - corporate_app/settings/* + + .. note:: + In earlier releases, this documentation incorrectly stated that + binary files would not affected by the ``keep_newline``. However, + this module does not actually distinguish between binary and text + files. + + + :param render_default: + Override Salt's :conf_master:`default global renderer ` for + the ``file_tree`` pillar. + + .. code-block:: yaml + + render_default: jinja + + :param renderer_blacklist: + Disallow renderers for pillar files. + + .. code-block:: yaml + + renderer_blacklist: + - json + + :param renderer_whitelist: + Allow renderers for pillar files. + + .. code-block:: yaml + + renderer_whitelist: + - yaml + - jinja + + :param template: + Enable templating of pillar files. Defaults to ``False``. ''' # Not used del pillar