Merge pull request #9784 from cro/proxyminion_rebase

Proxy minion support

doc/_static/proxy_minions.drawio.xml

@@ -0,0 +1 @@
+<mxfile><diagram>7V1bc9s2Fv41mWkfhCHu5KOdxulMm0622dnt7ouGlmiZU0p0KTp2+usLSgKFGy1QgkTarTpNIpDi5TvfueIQfIffL58/VunD/adynhXvUDR/fod/eIcQZIiJv5qRb7uRhCXbkUWVz3dju4HHfJ6ttaG6LIs6f9AHZ+Vqlc1qbeyuLPSDPaSLbDsQ7Qe+zNIis3b7bz6v77ejsbzcZvzHLF/cy9NAedm36ez3RVU+rnbne4fwzeaz3byuv8lTzLO79LGoJ5sh1GzGH97h91VZ1tt/LZ/fZ0UDlwRie6k3HVvbS66yVe31A8S3P/maFo+ZvFzjKsVPBMDiy/X6Pn1oBmdF+SiOcP10n9fZl4d01gw+CfHa599d0tesqrNnZWh3PR+zcpnV1Texy24rJgRQSjnhEDGCJag7clDOAER0O/a0FwuUQrxXRYJ3g+mOHov2bHtMxD92sLghkhfwEkKNtHvce0vQ9LZQKdiNCUsAjPYfSDRQMAckwZwk20/MiI0Pi0CcKJ/YhosGQItHh9Fa11X5e/a+LMpqM4DZ5qPwa/m8aGwFqIQmgUW2yqq0ALO0ptm0GZouhXLV04e0nt2LP1dZMUVk+lBWtbjya1sOrQgPk9CALHGQioA4AE7Qwmn+uLydZ1/zWWMJhJHBz6m4m+9WWf1UVr83ED7l4oa/t/AU9ySu7Tot8sWqUU5x35lA9rq53VzYs6vdhmU+nze/OQ0hSckIEKpQklCNklIdVTwjh4qG0FCORqChcQIoUdRL19CEghjt4YJsOAXFI1TQVoIjUlDygoLiiytoD4TGp6AsHoGCYgwwjmgSYxZHMeMaGhMUAaaAFSE8mIYym3kjcKFxhyCG01Bmh657DYWXd6H+CI1PQ+MxaCiPAFZVUEcjjgC0lU8FR/ye2vCQBOAYIumbeUIDwEU9VHSniPlyk2e2VPo5vc2Kz+U6r/OyodRtWdfl0sG1unwQo5ufX60ftuksbEgqv9zlz9lc7iK+39e1+MVVcxfopsZgsa5TcUQwE4dHN5u9xD3f/NEc+1ac4erql3nycfbrv/5//fXL5I/Zb7/i5CdGZz/ARX4P8yr680Pyy3ry24St/vPr1fMn/tOXP37+8ab6ABH++O8vcW2TIT5CCSxBEgQohCQhiKKYY+SwICgGRI22YAB7ErMX7IkQePRdjmbTormI9X35ZBuRTcrfSGQjJo/cuAdaMtODApp4n+k5Ej23FkAOYob2kNEQWuBROzjI+q0nkwUW1I4ovi2K4ugmElvu8qJQxu82H4fPy8s1B7kwP2uwEdftY3HrtNn96Zokus22KxEIAUdggBLAT0cc+kRS2XyRfdl9XZWrxvZkq/lVVZVP+5F5ur5vqdp8+ZzWwuWtNiMo2vGqOZRm0m3wXvb/Ln/VDlZZIezTV/0MLnh25/hc5uLcrSg4AUTYCk54RLGwFnqhKAEGw9flYzXLdsdASknMOGzS77B1Wi2y2jrsRpItKJ7C9ag0/Y2Ei+NYBNRCHoxGiR6uQwy4YsySKDla1iecJaDofcpm/4h+c7hEhIFR68o44mcR/YGzBBR9FMKLumNH3bdCt6cE+ef7LXduF5Jet4uPVZatbJebbj5OX9q70E/iBCQkEi4VxiQiONbTHxECOiv93BXdhMi7YeThXIeOZ5bZer2N40OIQEQygNKulIsyDhwCoNCdtmo5KQc8QIQJIw+XKBMsmQbly4U4bpHfij9nRf4wTau6SYiyIhOZ0PJxJeS3kRq6+SK0JS2mdfmUVdNyNYUofhb/g4fVIgC6mL00k8UiN79jh2mNQ9Ab+kzNGMyb5etZCZblPFuup+lqPn1obMUaVHfTzaBN4gizWBhTB+339DZUJADSVKQ3SsqoG3OSIIAVR26XDmJkY05QAMyJjfk+p+RqjWotAF5nVZ4WARLL/gAKhe2uvdBm64sAOqdfQxSmIOlXmfIuQzkA60SnjWxktRgn4ECxGLkiJKHzpyNC7TJFAMIQ/0xYFuQwQJHDDRwEATPA1TJ6AD9BPWKorvJl5533LmAinSWCJIcA8plSICE441OvPG5KAT5u5xOE7fq6KZnbELfSGc/cAZTWXUFknRb1ZJmvNqFBa5gfqnImYq5OTym3d6qZg1cO/vmSikFjnsrFKYfSJYDHfB+1SYt1Gob2VPIGQwHJ87fxIQn1Gfq2BCpTPoFRpKb7Di/HImf1T3N6GEB1lyQE0LbJv9vMZxlIuioEz3n9m/jaFIW33/638woCt+qbsqn52mzbsefFEkFrNLfZtqFT2+RYI8lw1YQJgroGwcRwXMfWCY3Dmv7wfHVCYnu7rbIJfyWMeFFkFbS4ceQM54k6R4TdUcyO1IZDwu+KLU7SIlkjeCXmCh2YHYYURPF+Bzwic2Un7a/ZL3AEoMphPhqgoUcvl+oTsuK2fPqwH7jeDIgN92WV/ymMR1ooLgMqDqOdU3RkmZoPkEqm+gDZcma4lOF8AhG6lSiygJrAYwywIybv6x+o4XestOd8DgLawe3QvJA2QeOFO9QYjhdNE2c3L6AwFGqNJDpy5oFwvRAo7Au/TOAAZV1oRLzANi9kA9x47AXTVXkik7LeojeCRrMe1CHrq6pKvym7PTQ7rF/gsZE0cu0hEPGP7QGP5hH14NEFanKAKW5ZdgC11Sk7Mz5fTQ5SD4vrVZQ7FMvQ11OUg9QjPAlAky5IRl+Vg5QcBihsWe4gvTyy9bPW5RyVyoHqcv1oNaa6nGOGYND86wCSr7cu55h2GLguJz2znpPt4NDqcluS/FOXCxheU3uq0qzL2fb9THW5Azo3qrrcyMpFBw3/a63LSQf0RvzCeOtyPk9wnTs/wnEEsPLIKdbAw4S7tqppAqcAqW7Xhq55qBXvJSAzrlOgS2xb8BJyGhwt6sdP5KvciYe4f4htH+ILQIvdq77/Ezo5WuxeNQAnaADuarR4RQBwj/ams7VDNy2iddX8UmTT87wE6aYjbzqZpiIsW63SKXI33uIu3eu0z0ZJEbsKNTbgZuHxKBvr0Qd6CYTTW7F7OqvB8vF5Ms/Enydjq6DnyraTAOjBqF9Vy+9WeveaJe7Ipp2qAIm6Xozsb1XwSRz4MAZCPEiZePiwSxDsKf2aiXx8uQbrfJVNm69uhrUi7cUwxAFMYhGiEhixJI4SdB59hfilB1OZ2kRMsLiT6NOPf26uVGToIj7/3sK+f3foEebN8B36VBpz+42LNBSzy1Slt9Fot/42+CizF0TXXwaokto4StQ8AWqFmnMbLUIEyMo+IaKTaCS+w1u12VGqLZJCHu/XCnFqtoFuCMMJ5YzzW4S3nbfTmR5Z3B4MfPTmwTcmLRwrjAwFPvPoUA/XmqBW2bUa+37bZxH5i9toarF+pffd5WqFd0fTS+KReJ217m4kHtQMir17GfTjEPM4oXoZjOslOHAzA/NIOEMy70guSRhULm2fmB2OS9jgAKbHcsk4kLWOVshVFnxWPA3bBNXfcDBb2Lyji/ViwuaGjI42HMaBoKflOGpdBXJY2GfvVIJ6ro70p1O7nnN39unAIE8Ccw9nG6RbiXckQSd2Kwn9CN+t5FN4DECVLkjG363EPQznZbuVuEcPw1m7lbhd6RqoW6kfrcbUrSS7occyK30AydfbrSRdx3i6laR31mIfqVNat9KWJANmTW+vWym2a01mtxK2uHGmbqUDOjembiW55OKY0wdpUjUVggMXHqyHEsygxDt/MJ2ZeaBwSuITKJ97WdxJwgBTn3RA2t2LvBDEilMmlm4kzcLXivYQh4wZAYjbXuukNSc8atlHhovbMHEzF1+VdbqrwSZNKa9oCrPX7QtYtIWJmv+aBYuE0inj6qtZtBXXaYcEXyhtAyEdZdrGsXhZIwsH/hS3K22dhLltnLbxaLpu7LQJv8fUXw8UpDoibWbLEVt6ETLM+mE+CzX3tdZH2WSpDfraCGhYkzwxFzbqX9O5QA3HsS5zCI/bOQEQTr506Pqs8BxIz08wo4DCvX7invKWJgDrxCEdbyA6Bx3OU7+9AB2GruASBx14GDoY9Xt6OTokHu0cs0cRL2xLmd7SVCXX1ah+KcEZAS+BR0bOExLrnld2FrXPCXPA4D6YhF5S6zujB91303nRL+9/+gSgT7uawqAgtmJUj2J1ANyfXrjJ05VlIqXat5EhBomSyBhJRiB6mfPSB+gV67vjJDS77PLmZdnFBs77TeN17NoJIuR4qWUOx1yraUvTFphd1H03nuwiO/hCsQv59OO9Ne93LIEmiAFKRDrBSRTxOJKr5MjjRs3zZHvzhPxiFm8CeTJm0nG74SjjUZc5r7sbuJZvurtjm1YwBRA6XqnTpkQUMKWL16zvhOqPMu6GvUwv037tXHQ4dqG3b5BCEWgCtxRqKWKsYUNEVq0+SnweAlnxNTlgoEzGBW7IQz4P2JzVQA0+2WhIxJrg8CUYg0BEIDihIk6gmCMjZGIcAqbETBeJx9lu0QzPiMnYPQC7yNs3UMEIZBWIBVv0IOk8JsnwUuxQzGTsTwMveBZ75HADvx0TxiCJkvaZKulJhnk9pqMtysLr7/l6zKSDDp1iR/aqFkO9HtPOG/ZPIQqjeobXY/qjJYsAo3o9pkccM/LXY/ama8frMb9Jd2LJ46xvy+Qeqe7F3qo3cFDJdathvNZy86rDhMnSP5QPjPduZzvpLCEb/z3y0H9Evzlc86pDiqFZnwgk8o6jhxS1h539O4macEowY5hyZKxgnQAakVb92nXZj5H40ScJJ3gU9WzuaQUftlww+BtOjGzPWtHYV65G5bnt8TiH6Hr29JxLdEOpqxSV+TiX92QYMh9D0EXXLCChbDU7n0NKsmc7zpkkSQee5TQfYD1WsBPzgQDzQCFF55GmW6I7SjxjK4gdbSIn0DSSZvE/pHw8yk6vXD7B1cXzjQrHiMPn/blvQ12CGTOjLtu+5v5k6YivVVnW6u5NaeNTOc+aPf4C</diagram></mxfile>

doc/contents.rst

@@ -34,6 +34,7 @@ Salt Table of Contents
     topics/development/index
     topics/translating
     topics/salt_projects
+    topics/proxyminion/index
 
     ref/configuration/logging/*
     ref/configuration/logging/handlers/*

doc/index.rst

@@ -219,6 +219,9 @@ Salt is many splendid things.
 :doc:`Testing Salt <topics/tests/index>`
     A howto for writing unit tests and integration tests.
 
+:doc:`Salt Proxy Minions <topics/proxyminion/index>`
+    Controlling devices and machines unable to run a salt-minion.
+
 :ref:`Python API interface <python-api>`
     Use Salt programmatically from scripts and programs easily and
     simply via ``import salt``.

doc/topics/proxyminion/index.rst

@@ -0,0 +1,286 @@
+===============================
+Salt Proxy Minion Documentation
+===============================
+
+Proxy minions are a developing Salt feature that enables controlling devices
+that, for whatever reason, cannot run a standard salt-minion.  Examples include
+network gear that has an API but runs a proprietary OS, devices with limited
+CPU or memory, or devices that could run a minion, but for security reasons,
+will not.
+
+*Proxy minions are not an "out of the box" feature*.  Because
+there are an infinite number of controllable devices,
+you will most likely have to write the
+interface yourself. Fortunately, this is only as difficult as the actual
+interface to the proxied device.  Devices that have an existing Python module
+(PyUSB for example) would be relatively simple to interface.  Code to control
+a device that has an HTML REST-based interface should be easy.  Code to control
+your typical housecat would be excellent source material for a PhD thesis.
+
+Salt proxy-minions provide the 'plumbing' that allows device enumeration
+and discovery, control, status, remote execution, and state management.
+
+Getting Started
+---------------
+
+The following diagram may be helpful in understanding the structure of a Salt
+installation that includes proxy-minions:
+
+.. image:: /_static/proxy_minions.png
+
+The key thing to remember is the left-most section of the diagram.  Salt's
+nature is to have a minion connect to a master, then the master may control
+the minion.  However, for proxy minions, the target device cannot run a minion,
+and thus must rely on a separate minion to fire up the proxy-minion and make the
+initial and persistent connection.
+
+After the proxy minion is started and initiates its connection to the 'dumb'
+device, it connects back to the salt-master and ceases to be affiliated in
+any way with the minion that started it.
+
+To create support for a proxied device one needs to create four things:
+
+1. The `proxytype connection class`_ (located in salt/proxy).
+2. The `grains support code`_ (located in salt/grains).
+3. `Salt modules`_ specific to the controlled device.
+4. _`Salt states` specific to the controlled device.
+
+
+Configuration parameters on the master
+######################################
+
+Proxy minions require no configuration parameters in /etc/salt/master.
+
+Salt's Pillar system is ideally suited for configuring proxy-minions.  Proxies
+can either be designated via a pillar file in pillar_roots, or through an
+external pillar.  External pillars afford the opportunity for interfacing with
+a configuration management system, database, or other knowledgeable system that
+that may already contain all the details of proxy targets.  To use static files
+in pillar_roots, pattern your files after the following examples, which are
+based on the diagram above:
+
+``/srv/salt/pillar/top.sls``
+
+.. code-block:: yaml
+
+    base:
+      minioncontroller1:
+        - networkswitches
+      minioncontroller2:
+        - reallydumbdevices
+      minioncontroller3:
+        - smsgateway
+
+
+``/srv/salt/pillar/networkswitches.sls``
+
+.. code-block:: yaml
+
+    proxy:
+      dumbdevice1:
+        proxytype: networkswitch
+        host: 172.23.23.5
+        username: root
+        passwd: letmein
+      dumbdevice2:
+        proxytype: networkswitch
+        host: 172.23.23.6
+        username: root
+        passwd: letmein
+      dumbdevice3:
+        proxytype: networkswitch
+        host: 172.23.23.7
+        username: root
+        passwd: letmein
+
+``/srv/salt/pillar/reallydumbdevices.sls``
+
+.. code-block:: yaml
+
+    proxy:
+      dumbdevice4:
+        proxytype: i2c_lightshow
+        i2c_address: 1
+      dumbdevice5:
+        proxytype: i2c_lightshow
+        i2c_address: 2
+      dumbdevice6:
+        proxytype: 433mhz_wireless
+
+``/srv/salt/pillar/smsgateway.sls``
+
+.. code-block:: yaml
+
+    proxy:
+      minioncontroller3:
+        dumbdevice7:
+          proxytype: sms_serial
+          deventry: /dev/tty04
+
+Note the contents of each minioncontroller key may differ widely based on
+the type of device that the proxy-minion is managing.
+
+In the above example
+
+- dumbdevices 1, 2, and 3 are network switches that have a management
+  interface available at a particular IP address.
+
+- dumbdevices 4 and 5 are very low-level devices controlled over an i2c bus.  In this case
+  the devices are physically connected to machine 'minioncontroller2', and are addressable
+  on the i2c bus at their respective i2c addresses.
+
+- dumbdevice6 is a 433 MHz wireless transmitter, also physically connected to minioncontroller2
+
+- dumbdevice7 is an SMS gateway connected to machine minioncontroller3 via a serial port.
+
+Because of the way pillar works, each of the salt-minions that fork off the
+proxy minions will only see the keys specific to the proxies it will be
+handling.  In other words, from the above example, only minioncontroller1 will
+see the connection information for dumbdevices 1, 2, and 3.  Minioncontroller2
+will see configuration data for dumbdevices 4, 5, and 6, and minioncontroller3
+will be privy to dumbdevice7.
+
+Also, in general, proxy-minions are lightweight, so the machines that run them
+could conceivably control a large number of devices.  The example above is just
+to illustrate that it is possible for the proxy services to be spread across
+many machines if necessary, or intentionally run on machines that need to
+control devices because of some physical interface (e.g. i2c and serial above).
+Another reason to divide proxy services might be security.  In more secure
+environments only certain machines may have a network path to certain devices.
+
+Now our salt-minions know if they are supposed to spawn a proxy-minion process
+to control a particular device.  That proxy-minion process will initiate
+a connection back to the master to enable control.
+
+
+.. _proxytype connection class:
+
+Proxytypes
+##########
+
+A proxytype is a Python class called 'Proxyconn' that encapsulates all the code
+necessary to interface with a device.  Proxytypes are located inside the
+salt.proxy module.  At a minimum a proxytype object must implement the
+following methods:
+
+``proxytype(self)``: Returns a string with the name of the proxy type.
+
+``proxyconn(self, **kwargs)``: Provides the primary way to connect and communicate
+with the device. Some proxyconns instantiate a particular object that opens a
+network connection to a device and leaves the connection open for communication.
+Others simply abstract a serial connection or even implement endpoints to communicate
+via REST over HTTP.
+
+``id(self, opts)``: Returns a unique, unchanging id for the controlled device.  This is
+the "name" of the device, and is used by the salt-master for targeting and key
+authentication.
+
+Optionally, the class may define a ``shutdown(self, opts)`` method if the
+controlled device should be informed when the minion goes away cleanly.
+
+It is highly recommended that the ``test.ping`` execution module also be defined
+for a proxytype. The code for ``ping`` should contact the controlled device and make
+sure it is really available.
+
+Here is an example proxytype used to interface to Juniper Networks devices that run
+the Junos operating system.  Note the additional library requirements--most of the
+"hard part" of talking to these devices is handled by the jnpr.junos, jnpr.junos.utils
+and jnpr.junos.cfg modules.
+
+
+.. code-block:: python
+
+    # Import python libs
+    import logging
+    import os
+
+    import jnpr.junos
+    import jnpr.junos.utils
+    import jnpr.junos.cfg
+    HAS_JUNOS = True
+
+    class Proxyconn(object):
+
+
+        def __init__(self, details):
+            self.conn = jnpr.junos.Device(user=details['username'], host=details['host'], password=details['passwd'])
+            self.conn.open()
+            self.conn.bind(cu=jnpr.junos.cfg.Resource)
+
+
+        def proxytype(self):
+            return 'junos'
+
+
+        def id(self, opts):
+            return self.conn.facts['hostname']
+
+
+        def ping(self):
+            return self.conn.connected
+
+
+        def shutdown(self, opts):
+
+            print('Proxy module {} shutting down!!'.format(opts['id']))
+            try:
+                self.conn.close()
+            except Exception:
+                pass
+
+
+.. _grains support code:
+
+Grains are data about minions.  Most proxied devices will have a paltry amount
+of data as compared to a typical Linux server.  Because proxy-minions are
+started by a regular minion, they inherit a sizeable number of grain settings
+which can be useful, especially when targeting (PYTHONPATH, for example).
+
+All proxy minions set a grain called 'proxy'.  If it is present, you know the
+minion is controlling another device.  To add more grains to your proxy minion
+for a particular device, create a file in salt/grains named [proxytype].py and
+place inside it the different functions that need to be run to collect the data
+you are interested in.  Here's an example:
+
+
+
+The __proxyenabled__ directive
+------------------------------
+
+Salt states and execution modules, by and large, cannot "automatically" work
+with proxied devices.  Execution modules like ``pkg`` or ``sqlite3`` have no
+meaning on a network switch or a housecat.  For a state/execution module to be
+available to a proxy-minion, the ``__proxyenabled__`` variable must be defined
+in the module as an array containing the names of all the proxytypes that this
+module can support.  The array can contain the special value ``*`` to indicate
+that the module supports all proxies.
+
+If no ``__proxyenabled__`` variable is defined, then by default, the
+state/execution module is unavailable to any proxy.
+
+Here is an excerpt from a module that was modified to support proxy-minions:
+
+.. code-block:: python
+
+    def ping():
+
+        if 'proxyobject' in __opts__:
+            if 'ping' in __opts__['proxyobject'].__attr__():
+                return __opts['proxyobject'].ping()
+            else:
+                return False
+        else:
+            return True
+
+And then in salt.proxy.junos we find
+
+.. code-block:: python
+
+     def ping(self):
+
+        return self.connected
+
+
+The Junos API layer lacks the ability to do a traditional 'ping', so the
+example simply checks the connection object field that indicates
+if the ssh connection was successfully made to the device.

salt/__init__.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8 -*-
 '''
 Make me some salt!
 '''
@@ -235,6 +235,114 @@ class Minion(parsers.MinionOptionParser):
         '''
 
 
+class ProxyMinion(parsers.MinionOptionParser):
+    '''
+    Create a proxy minion server
+    '''
+    def prepare(self, proxydetails):
+        '''
+        Run the preparation sequence required to start a salt minion.
+
+        If sub-classed, don't **ever** forget to run:
+
+            super(YourSubClass, self).prepare()
+        '''
+        self.parse_args()
+
+        try:
+            if self.config['verify_env']:
+                confd = self.config.get('default_include')
+                if confd:
+                    # If 'default_include' is specified in config, then use it
+                    if '*' in confd:
+                        # Value is of the form "minion.d/*.conf"
+                        confd = os.path.dirname(confd)
+                    if not os.path.isabs(confd):
+                        # If configured 'default_include' is not an absolute
+                        # path, consider it relative to folder of 'conf_file'
+                        # (/etc/salt by default)
+                        confd = os.path.join(
+                            os.path.dirname(self.config['conf_file']), confd
+                        )
+                else:
+                    confd = os.path.join(
+                        os.path.dirname(self.config['conf_file']), 'minion.d'
+                    )
+                verify_env(
+                    [
+                        self.config['pki_dir'],
+                        self.config['cachedir'],
+                        self.config['sock_dir'],
+                        self.config['extension_modules'],
+                        confd,
+                    ],
+                    self.config['user'],
+                    permissive=self.config['permissive_pki_access'],
+                    pki_dir=self.config['pki_dir'],
+                )
+                if 'proxy_log' in proxydetails:
+                    logfile = proxydetails['proxy_log']
+                else:
+                    logfile = None
+                if logfile is not None and not logfile.startswith(('tcp://',
+                                                                   'udp://',
+                                                                   'file://')):
+                    # Logfile is not using Syslog, verify
+                    verify_files([logfile], self.config['user'])
+        except OSError as err:
+            sys.exit(err.errno)
+
+        self.config['proxy'] = proxydetails
+        self.setup_logfile_logger()
+        logger.info(
+            'Setting up a Salt Proxy Minion "{0}"'.format(
+                self.config['id']
+            )
+        )
+        migrations.migrate_paths(self.config)
+        # Late import so logging works correctly
+        import salt.minion
+        # If the minion key has not been accepted, then Salt enters a loop
+        # waiting for it, if we daemonize later then the minion could halt
+        # the boot process waiting for a key to be accepted on the master.
+        # This is the latest safe place to daemonize
+        self.daemonize_if_required()
+        self.set_pidfile()
+        if isinstance(self.config.get('master'), list):
+            self.minion = salt.minion.MultiMinion(self.config)
+        else:
+            self.minion = salt.minion.ProxyMinion(self.config)
+
+    def start(self, proxydetails):
+        '''
+        Start the actual minion.
+
+        If sub-classed, don't **ever** forget to run:
+
+            super(YourSubClass, self).start()
+
+        NOTE: Run any required code before calling `super()`.
+        '''
+        self.prepare(proxydetails)
+        try:
+            self.minion.tune_in()
+        except (KeyboardInterrupt, SaltSystemExit) as exc:
+            logger.warn('Stopping the Salt Proxy Minion')
+            if isinstance(exc, KeyboardInterrupt):
+                logger.warn('Exiting on Ctrl-c')
+            else:
+                logger.error(str(exc))
+        finally:
+            self.shutdown()
+
+    def shutdown(self):
+        '''
+        If sub-classed, run any shutdown operations on this method.
+        '''
+        if 'proxy' in self.minion.opts:
+            self.minion.opts['proxyobject'].shutdown(self.minion.opts)
+
+
 class Syndic(parsers.SyndicOptionParser):
     '''
     Create a syndic server

salt/config.py

@@ -181,6 +181,7 @@ VALID_OPTS = {
     'sign_pub_messages': bool,
     'keysize': int,
     'salt_transport': str,
+    'enumerate_proxy_minions': bool,
 }
 
 # default configurations
@@ -383,6 +384,7 @@ DEFAULT_MASTER_OPTS = {
     'sign_pub_messages': False,
     'keysize': 4096,
     'salt_transport': 'zeromq',
+    'enumerate_proxy_minions': False
 }
 
 # ----- Salt Cloud Configuration Defaults ----------------------------------->

salt/grains/core.py

@@ -38,7 +38,6 @@ __salt__ = {
     'cmd.run': salt.modules.cmdmod._run_quiet,
     'cmd.run_all': salt.modules.cmdmod._run_all_quiet
 }
-
 log = logging.getLogger(__name__)
 
 HAS_WMI = False
@@ -778,6 +777,7 @@ def os_data():
     # ('Linux', 'MINIONNAME', '2.6.32-38-server', '#83-Ubuntu SMP Wed Jan 4 11:26:59 UTC 2012', 'x86_64', '')
     (grains['kernel'], grains['nodename'],
      grains['kernelrelease'], version, grains['cpuarch'], _) = platform.uname()
+
     if salt.utils.is_windows():
         grains['osrelease'] = grains['kernelrelease']
         grains['osversion'] = grains['kernelrelease'] = version
@@ -995,6 +995,10 @@ def locale_info():
         defaultencoding
     '''
     grains = {}
+
+    if 'proxyminion' in __opts__:
+        return grains
+
     try:
         (grains['defaultlanguage'], grains['defaultencoding']) = locale.getdefaultlocale()
     except Exception:
@@ -1016,6 +1020,10 @@ def hostname():
     #   localhost
     #   domain
     grains = {}
+
+    if 'proxyminion' in __opts__:
+        return grains
+
     grains['localhost'] = socket.gethostname()
     if '.' in socket.getfqdn():
         grains['fqdn'] = socket.getfqdn()
@@ -1029,7 +1037,12 @@ def append_domain():
     '''
     Return append_domain if set
     '''
+
     grain = {}
+
+    if 'proxyminion' in __opts__:
+        return grain
+
     if 'append_domain' in __opts__:
         grain['append_domain'] = __opts__['append_domain']
     return grain
@@ -1039,6 +1052,10 @@ def ip4():
     '''
     Return a list of ipv4 addrs
     '''
+
+    if 'proxyminion' in __opts__:
+        return {}
+
     return {'ipv4': salt.utils.network.ip_addrs(include_loopback=True)}
 
 
@@ -1046,6 +1063,10 @@ def fqdn_ip4():
     '''
     Return a list of ipv4 addrs of fqdn
     '''
+
+    if 'proxyminion' in __opts__:
+        return {}
+
     try:
         info = socket.getaddrinfo(hostname()['fqdn'], None, socket.AF_INET)
         addrs = list(set(item[4][0] for item in info))
@@ -1058,6 +1079,10 @@ def ip6():
     '''
     Return a list of ipv6 addrs
     '''
+
+    if 'proxyminion' in __opts__:
+        return {}
+
     return {'ipv6': salt.utils.network.ip_addrs6(include_loopback=True)}
 
 
@@ -1065,6 +1090,10 @@ def fqdn_ip6():
     '''
     Return a list of ipv6 addrs of fqdn
     '''
+
+    if 'proxyminion' in __opts__:
+        return {}
+
     try:
         info = socket.getaddrinfo(hostname()['fqdn'], None, socket.AF_INET6)
         addrs = list(set(item[4][0] for item in info))
@@ -1079,6 +1108,10 @@ def ip_interfaces():
     '''
     # Provides:
     #   ip_interfaces
+
+    if 'proxyminion' in __opts__:
+        return {}
+
     ret = {}
     ifaces = salt.utils.network.interfaces()
     for face in ifaces:
@@ -1113,6 +1146,7 @@ def path():
     '''
     # Provides:
     #   path
+
     return {'path': os.environ['PATH'].strip()}
 
 
@@ -1176,6 +1210,9 @@ def _dmidecode_data(regex_dict):
     '''
     ret = {}
 
+    if 'proxyminion' in __opts__:
+        return {}
+
     # No use running if dmidecode/smbios isn't in the path
     if salt.utils.which('dmidecode'):
         out = __salt__['cmd.run']('dmidecode')
@@ -1231,6 +1268,10 @@ def _hw_data(osdata):
 
     .. versionadded:: 0.9.5
     '''
+
+    if 'proxyminion' in __opts__:
+        return {}
+
     grains = {}
     # TODO: *BSD dmidecode output
     if osdata['kernel'] == 'Linux':
@@ -1307,6 +1348,10 @@ def _smartos_zone_data():
     # Provides:
     #   pkgsrcversion
     #   imageversion
+
+    if 'proxyminion' in __opts__:
+        return {}
+
     grains = {}
 
     pkgsrcversion = re.compile('^release:\\s(.+)')
@@ -1338,6 +1383,9 @@ def get_server_id():
     '''
     # Provides:
     #   server_id
+
+    if 'proxyminion' in __opts__:
+        return {}
     return {'server_id': abs(hash(__opts__.get('id', '')) % (2 ** 31))}
 
 

salt/grains/junos.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+'''
+Grains for junos.
+NOTE this is a little complicated--junos can only be accessed via salt-proxy-minion.
+Thus, some grains make sense to get them from the minion (PYTHONPATH), but others
+don't (ip_interfaces)
+'''
+__proxyenabled__ = ['junos']
+
+__virtualname__ = 'junos'
+
+
+def __virtual__():
+    if 'proxyconn' not in __opts__:
+        return False
+    else:
+        return __virtualname__
+
+
+def location():
+    return {'location': 'dc-1-europe'}
+
+
+def os_family():
+    return {'os_family': 'junos'}
+
+
+def os_data():
+    facts = __opts__['proxyconn'].facts
+    facts['version_info'] = {'major': '12,1', 'type': 'I', 'minor': '20131108_srx_12q1_x46_intgr', 'build': '0-613414'}
+    facts['os_family'] = 'proxy'
+    return facts

salt/loader.py

@@ -127,6 +127,16 @@ def raw_mod(opts, name, functions):
     return load.gen_module(name, functions)
 
 
+def proxy(opts, functions, whitelist=None):
+    '''
+    Returns the proxy module for this salt-proxy-minion
+    '''
+    load = _create_loader(opts, 'proxy', 'proxy')
+    pack = {'name': '__proxy__',
+            'value': functions}
+    return load.gen_functions(pack, whitelist=whitelist)
+
+
 def returners(opts, functions, whitelist=None):
     '''
     Returns the returner modules
@@ -728,6 +738,20 @@ class Loader(object):
             modules.append(mod)
         for mod in modules:
             virtual = ''
+
+            # If this is a proxy minion then MOST modules cannot work.  Therefore, require that
+            # any module that does work with salt-proxy-minion define __proxyenabled__ as a list
+            # containing the names of the proxy types that the module supports.
+            if 'proxytype' in self.opts:
+                if not hasattr(mod, '__proxyenabled__'):
+                    # This is a proxy minion but this module doesn't support proxy
+                    # minions at all
+                    continue
+                if not (self.opts['proxytype'] in mod.__proxyenabled__ or '*' in mod.__proxyenabled__):
+                    # This is a proxy minion, this module supports proxy
+                    # minions, but not this particular minion
+                    continue
+
             if hasattr(mod, '__opts__'):
                 mod.__opts__.update(self.opts)
             else:

salt/minion.py

@@ -19,6 +19,8 @@ import traceback
 import sys
 import signal
 from random import randint
+import salt
+import importlib
 
 # Import third party libs
 try:
@@ -528,17 +530,33 @@ class Minion(object):
             opts['environment'],
         ).compile_pillar()
         self.serial = salt.payload.Serial(self.opts)
-        self.mod_opts = self.__prep_mod_opts()
-        self.functions, self.returners = self.__load_modules()
+        self.mod_opts = self._prep_mod_opts()
+        self.functions, self.returners = self._load_modules()
         self.matcher = Matcher(self.opts, self.functions)
         self.proc_dir = get_proc_dir(opts['cachedir'])
         self.schedule = salt.utils.schedule.Schedule(
             self.opts,
             self.functions,
             self.returners)
+
         self.grains_cache = self.opts['grains']
 
-    def __prep_mod_opts(self):
+        if 'proxy' in self.opts['pillar']:
+            log.debug('I am {0} and I need to start some proxies for {0}'.format(self.opts['id'],
+                                                                                 self.opts['pillar']['proxy']))
+            for p in self.opts['pillar']['proxy']:
+                log.debug('Starting {0} proxy.'.format(p))
+                pid = os.fork()
+                if pid > 0:
+                    continue
+                else:
+                    proxyminion = salt.ProxyMinion()
+                    proxyminion.start(self.opts['pillar']['proxy'][p])
+                    self.clean_die(signal.SIGTERM, None)
+        else:
+            log.debug("I am {0} and I am not supposed to start any proxies.".format(self.opts['id']))
+
+    def _prep_mod_opts(self):
         '''
         Returns a copy of the opts with key bits stripped out
         '''
@@ -549,7 +567,7 @@ class Minion(object):
             mod_opts[key] = val
         return mod_opts
 
-    def __load_modules(self):
+    def _load_modules(self):
         '''
         Return the functions and the returners loaded up from the loader
         module
@@ -693,7 +711,7 @@ class Minion(object):
         '''
         if isinstance(data['fun'], string_types):
             if data['fun'] == 'sys.reload_modules':
-                self.functions, self.returners = self.__load_modules()
+                self.functions, self.returners = self._load_modules()
                 self.schedule.functions = self.functions
                 self.schedule.returners = self.returners
         if isinstance(data['fun'], tuple) or isinstance(data['fun'], list):
@@ -1023,7 +1041,7 @@ class Minion(object):
         '''
         Refresh the functions and returners.
         '''
-        self.functions, self.returners = self.__load_modules()
+        self.functions, self.returners = self._load_modules()
         self.schedule.functions = self.functions
         self.schedule.returners = self.returners
 
@@ -1828,3 +1846,62 @@ class Matcher(object):
                 salt.utils.minions.nodegroup_comp(tgt, nodegroups)
             )
         return False
+
+
+class ProxyMinion(Minion):
+    '''
+    This class instantiates a 'proxy' minion--a minion that does not manipulate
+    the host it runs on, but instead manipulates a device that cannot run a minion.
+    '''
+    def __init__(self, opts, timeout=60, safe=True):
+        '''
+        Pass in the options dict
+        '''
+
+        # Warn if ZMQ < 3.2
+        if HAS_ZMQ and (not(hasattr(zmq, 'zmq_version_info')) or
+                        zmq.zmq_version_info() < (3, 2)):
+            # PyZMQ 2.1.9 does not have zmq_version_info
+            log.warning('You have a version of ZMQ less than ZMQ 3.2! There '
+                        'are known connection keep-alive issues with ZMQ < '
+                        '3.2 which may result in loss of contact with '
+                        'minions. Please upgrade your ZMQ!')
+        # Late setup the of the opts grains, so we can log from the grains
+        # module
+        # print opts['proxymodule']
+        fq_proxyname = 'proxy.'+opts['proxy']['proxytype']
+        self.proxymodule = salt.loader.proxy(opts, fq_proxyname)
+        opts['proxyobject'] = self.proxymodule[opts['proxy']['proxytype']+'.Proxyconn'](opts['proxy'])
+        opts['id'] = opts['proxyobject'].id(opts)
+        opts.update(resolve_dns(opts))
+        self.opts = opts
+        self.authenticate(timeout, safe)
+        self.opts['pillar'] = salt.pillar.get_pillar(
+            opts,
+            opts['grains'],
+            opts['id'],
+            opts['environment'],
+        ).compile_pillar()
+        self.serial = salt.payload.Serial(self.opts)
+        self.mod_opts = self._prep_mod_opts()
+        self.functions, self.returners = self._load_modules()
+        self.matcher = Matcher(self.opts, self.functions)
+        self.proc_dir = get_proc_dir(opts['cachedir'])
+        self.schedule = salt.utils.schedule.Schedule(
+            self.opts,
+            self.functions,
+            self.returners)
+        self.grains_cache = self.opts['grains']
+
+    def _prep_mod_opts(self):
+        '''
+        Returns a copy of the opts with key bits stripped out
+        '''
+        return super(ProxyMinion, self)._prep_mod_opts()
+
+    def _load_modules(self):
+        '''
+        Return the functions and the returners loaded up from the loader
+        module
+        '''
+        return super(ProxyMinion, self)._load_modules()

salt/modules/apt.py

@@ -53,7 +53,7 @@ def __virtual__():
     '''
     Confirm this module is on a Debian based system
     '''
-    if __grains__['os_family'] != 'Debian':
+    if __grains__.get('os_family', False) != 'Debian':
         return False
     return __virtualname__
 

salt/modules/config.py

@@ -12,6 +12,8 @@ import urllib2
 import salt.utils
 import salt.syspaths as syspaths
 
+__proxyenabled__ = ['*']
+
 # Set up the default values for all systems
 DEFAULTS = {'mongo.db': 'salt',
             'mongo.host': 'salt',

salt/modules/grains.py

@@ -16,6 +16,8 @@ import salt.utils
 import salt.utils.dictupdate
 from salt.exceptions import SaltException
 
+__proxyenabled__ = ['*']
+
 # Seed the grains dict so cython will build
 __grains__ = {}
 

salt/modules/junos.py

@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+'''
+Module for interfacing to Junos devices
+
+ALPHA QUALITY code.
+
+'''
+
+# Import python libraries
+import re
+import logging
+
+# Salt libraries
+import salt.roster
+
+# Juniper interface libraries
+# https://github.com/jeremyschulman/py-junos-eznc
+
+
+try:
+    import jnpr.junos
+    import jnpr.junos.utils
+    import jnpr.junos.cfg
+    HAS_JUNOS = True
+except ImportError:
+    HAS_JUNOS = False
+
+
+# Set up logging
+log = logging.getLogger(__name__)
+
+# Define the module's virtual name
+__virtualname__ = 'junos'
+
+__proxyenabled__ = ['junos']
+
+
+def __virtual__():
+    '''
+    We need the Junos adapter libraries for this
+    module to work.  We also need a proxyconn object
+    in the opts dictionary
+    '''
+    if HAS_JUNOS and 'proxyconn' in __opts__:
+        return __virtualname__
+    else:
+        return False
+
+
+def facts_refresh():
+    '''
+    Reload the facts dictionary from the device.  Usually only needed
+    if the device configuration is changed by some other actor.
+    '''
+
+    return __opts__['proxyconn'].refresh
+
+
+def set_hostname(hostname=None, commit=True):
+
+    ret = dict()
+    conn = __opts__['proxyconn']
+    if hostname is None:
+        ret['out'] = False
+        return ret
+
+    # Added to recent versions of JunOs
+    # Use text format instead
+    set_string = 'set system host-name {0}'.format(hostname)
+
+    conn.cu.load(set_string, format='set')
+    if commit:
+        return commit()
+    else:
+        ret['out'] = True
+        ret['msg'] = 'set system host-name {0} is queued'.format(hostname)
+
+    return ret
+
+
+def commit():
+
+    conn = __opts__['proxyconn']
+
+    ret = {}
+    commit_ok = conn.cu.commit_check()
+    if commit_ok:
+        try:
+            conn.cu.commit(confirm=True)
+            ret['out'] = True
+            ret['message'] = 'Commit Successful.'
+        except Exception as e:
+            ret['out'] = False
+            ret['message'] = 'Pre-commit check succeeded but actual commit failed with "{0}"'.format(e.message)
+    else:
+        ret['out'] = False
+        ret['message'] = 'Pre-commit check failed.'
+
+    return ret
+
+
+def rollback():
+    conn = __opts__['proxyconn']
+    ret = dict()
+
+    ret['out'] = conn.cu.rollback(0)
+
+    if ret['out']:
+        ret['message'] = 'Rollback successful'
+    else:
+        ret['message'] = 'Rollback failed'
+
+    return ret
+
+
+def diff():
+
+    ret = dict()
+    conn = __opts__['proxyconn']
+    ret['out'] = True
+    ret['message'] = conn.cu.diff()
+
+    return ret
+
+
+def ping():
+
+    ret = dict()
+    conn = __opts__['proxyconn']
+    ret['message'] = conn.cli('show system uptime')
+    ret['out'] = True

salt/modules/mine.py

@@ -11,6 +11,7 @@ import logging
 import salt.crypt
 import salt.payload
 
+__proxyenabled__ = ['junos']
 
 log = logging.getLogger(__name__)
 

salt/modules/test.py

@@ -14,6 +14,8 @@ import salt
 import salt.version
 import salt.loader
 
+__proxyenabled__ = ['*']
+
 
 def echo(text):
     '''
@@ -39,7 +41,11 @@ def ping():
 
         salt '*' test.ping
     '''
-    return True
+
+    if 'proxyobject' in __opts__:
+        return __opts__['proxyobject'].ping()
+    else:
+        return True
 
 
 def sleep(length):

salt/proxy/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

salt/proxy/junos.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+'''
+Routines to set up a minion
+'''
+
+# Import python libs
+import logging
+import os
+
+import jnpr.junos
+import jnpr.junos.utils
+import jnpr.junos.cfg
+HAS_JUNOS = True
+
+
+class Proxyconn(object):
+
+    def __init__(self, details):
+        self.conn = jnpr.junos.Device(user=details['username'], host=details['host'], password=details['passwd'])
+        self.conn.open()
+        self.conn.bind(cu=jnpr.junos.cfg.Resource)
+
+    def proxytype(self):
+        return 'junos'
+
+    def id(self, opts):
+        return self.conn.facts['hostname']
+
+    def ping(self):
+        return self.conn.connected
+
+    def shutdown(self, opts):
+
+        print('Proxy module {0} shutting down!!'.format(opts['id']))
+        try:
+            self.conn.close()
+        except Exception:
+            pass

salt/utils/__init__.py

@@ -1164,9 +1164,17 @@ def is_windows():
 @real_memoize
 def is_linux():
     '''
-    Simple function to return if a host is Linux or not
+    Simple function to return if a host is Linux or not.
+    Note for a proxy minion, we need to return something else
     '''
-    return sys.platform.startswith('linux')
+    import __main__ as main
+    # This is a hack.  If a proxy minion is started by other
+    # means, e.g. a custom script that creates the minion objects
+    # then this will fail.
+    if ('salt-proxy-minion' in main.__file__):
+        return False
+    else:
+        return sys.platform.startswith('linux')
 
 
 @real_memoize

salt/utils/minions.py

@@ -407,7 +407,7 @@ class CkMinions(object):
                 grains = self.serial.load(
                     salt.utils.fopen(datap)
                 ).get('grains')
-                for ipv4 in grains['ipv4']:
+                for ipv4 in grains.get('ipv4', []):
                     if ipv4 == '127.0.0.1' or ipv4 == '0.0.0.0':
                         continue
                     if ipv4 in addrs: