Merge pull request #41398 from rallytime/merge-2016.11

[2016.11] Merge forward from 2016.3 to 2016.11

doc/ref/cli/salt-cp.rst

@@ -2,32 +2,42 @@
 ``salt-cp``
 ===========
 
-Copy a file to a set of systems
+Copy a file or files to one or more minions
 
 Synopsis
 ========
 
 .. code-block:: bash
 
-    salt-cp '*' [ options ] SOURCE DEST
+    salt-cp '*' [ options ] SOURCE [SOURCE2 SOURCE3 ...] DEST
 
-    salt-cp -E '.*' [ options ] SOURCE DEST
+    salt-cp -E '.*' [ options ] SOURCE [SOURCE2 SOURCE3 ...] DEST
 
-    salt-cp -G 'os:Arch.*' [ options ] SOURCE DEST
+    salt-cp -G 'os:Arch.*' [ options ] SOURCE [SOURCE2 SOURCE3 ...] DEST
 
 Description
 ===========
 
-Salt copy copies a local file out to all of the Salt minions matched by the
+salt-cp copies files from the master to all of the Salt minions matched by the
-given target.
+specified target expression.
 
-Salt copy is only intended for use with small files (< 100KB). If you need
+.. note::
-to copy large files out to minions please use the cp.get_file function.
+    salt-cp uses Salt's publishing mechanism. This means the privacy of the
+    contents of the file on the wire is completely dependent upon the transport
+    in use. In addition, if the master or minion is running with debug logging,
+    the contents of the file will be logged to disk.
 
-Note: salt-cp uses salt's publishing mechanism. This means the privacy of the
+    In addition, this tool is less efficient than the Salt fileserver when
-contents of the file on the wire is completely dependent upon the transport
+    copying larger files. It is recommended to instead use
-in use. In addition, if the salt-master is running with debug logging it is
+    :py:func:`cp.get_file <salt.modules.cp.get_file>` to copy larger files to
-possible that the contents of the file will be logged to disk.
+    minions. However, this requires the file to be located within one of the
+    fileserver directories.
+
+.. versionchanged:: 2016.3.7,2016.11.6,Nitrogen
+    Compression support added, disable with ``-n``. Also, if the destination
+    path ends in a path separator (i.e. ``/``,  or ``\`` on Windows, the
+    desitination will be assumed to be a directory. Finally, recursion is now
+    supported, allowing for entire directories to be copied.
 
 Options
 =======
@@ -46,6 +56,12 @@ Options
 .. include:: _includes/target-selection.rst
 
 
+.. option:: -n, --no-compression
+
+    Disable gzip compression.
+
+    .. versionadded:: 2016.3.7,2016.11.6,Nitrogen
+
 See also
 ========
 

salt/cli/cp.py

@@ -9,15 +9,26 @@ Salt-cp can be used to distribute configuration files
 # Import python libs
 from __future__ import print_function
 from __future__ import absolute_import
+import base64
+import errno
+import logging
 import os
+import re
 import sys
 
 # Import salt libs
 import salt.client
-from salt.utils import parsers, print_cli
+import salt.utils.gzip_util
+import salt.utils.minions
+from salt.utils import parsers, to_bytes
 from salt.utils.verify import verify_log
 import salt.output
 
+# Import 3rd party libs
+from salt.ext import six
+
+log = logging.getLogger(__name__)
+
 
 class SaltCPCli(parsers.SaltCPOptionParser):
     '''
@@ -44,65 +55,168 @@ class SaltCP(object):
     '''
     def __init__(self, opts):
         self.opts = opts
+        self.is_windows = salt.utils.is_windows()
 
-    def _file_dict(self, fn_):
+    def _mode(self, path):
-        '''
+        if self.is_windows:
-        Take a path and return the contents of the file as a string
+            return None
-        '''
+        try:
-        if not os.path.isfile(fn_):
+            return int(oct(os.stat(path).st_mode)[-4:], 8)
-            err = 'The referenced file, {0} is not available.'.format(fn_)
+        except (TypeError, IndexError, ValueError):
-            sys.stderr.write(err + '\n')
+            return None
-            sys.exit(42)
-        with salt.utils.fopen(fn_, 'r') as fp_:
-            data = fp_.read()
-        return {fn_: data}
 
-    def _recurse_dir(self, fn_, files=None):
+    def _recurse(self, path):
         '''
-        Recursively pull files from a directory
+        Get a list of all specified files
-        '''
-        if files is None:
-            files = {}
-
-        for base in os.listdir(fn_):
-            path = os.path.join(fn_, base)
-            if os.path.isdir(path):
-                files.update(self._recurse_dir(path))
-            else:
-                files.update(self._file_dict(path))
-        return files
-
-    def _load_files(self):
-        '''
-        Parse the files indicated in opts['src'] and load them into a python
-        object for transport
         '''
         files = {}
+        empty_dirs = []
+        try:
+            sub_paths = os.listdir(path)
+        except OSError as exc:
+            if exc.errno == errno.ENOENT:
+                # Path does not exist
+                sys.stderr.write('{0} does not exist\n'.format(path))
+                sys.exit(42)
+            elif exc.errno in (errno.EINVAL, errno.ENOTDIR):
+                # Path is a file (EINVAL on Windows, ENOTDIR otherwise)
+                files[path] = self._mode(path)
+        else:
+            if not sub_paths:
+                empty_dirs.append(path)
+            for fn_ in sub_paths:
+                files_, empty_dirs_ = self._recurse(os.path.join(path, fn_))
+                files.update(files_)
+                empty_dirs.extend(empty_dirs_)
+
+        return files, empty_dirs
+
+    def _list_files(self):
+        files = {}
+        empty_dirs = set()
         for fn_ in self.opts['src']:
-            if os.path.isfile(fn_):
+            files_, empty_dirs_ = self._recurse(fn_)
-                files.update(self._file_dict(fn_))
+            files.update(files_)
-            elif os.path.isdir(fn_):
+            empty_dirs.update(empty_dirs_)
-                print_cli(fn_ + ' is a directory, only files are supported.')
+        return files, sorted(empty_dirs)
-                #files.update(self._recurse_dir(fn_))
-        return files
 
     def run(self):
         '''
         Make the salt client call
         '''
-        arg = [self._load_files(), self.opts['dest']]
+        files, empty_dirs = self._list_files()
+        dest = self.opts['dest']
+        gzip = self.opts['gzip']
+        tgt = self.opts['tgt']
+        timeout = self.opts['timeout']
+        selected_target_option = self.opts.get('selected_target_option')
+
+        dest_is_dir = bool(empty_dirs) \
+            or len(files) > 1 \
+            or bool(re.search(r'[\\/]$', dest))
+
+        reader = salt.utils.gzip_util.compress_file \
+            if gzip \
+            else salt.utils.itertools.read_file
+
+        minions = salt.utils.minions.CkMinions(self.opts).check_minions(
+            tgt,
+            expr_form=selected_target_option or 'glob')
+
         local = salt.client.get_local_client(self.opts['conf_file'])
-        args = [self.opts['tgt'],
+
-                'cp.recv',
+        def _get_remote_path(fn_):
-                arg,
+            if fn_ in self.opts['src']:
-                self.opts['timeout'],
+                # This was a filename explicitly passed on the CLI
+                return os.path.join(dest, os.path.basename(fn_)) \
+                    if dest_is_dir \
+                    else dest
+            else:
+                for path in self.opts['src']:
+                    relpath = os.path.relpath(fn_, path + os.sep)
+                    if relpath.startswith(parent):
+                        # File is not within this dir
+                        continue
+                    return os.path.join(dest, os.path.basename(path), relpath)
+                else:  # pylint: disable=useless-else-on-loop
+                    # Should not happen
+                    log.error('Failed to find remote path for %s', fn_)
+                    return None
+
+        ret = {}
+        parent = '..' + os.sep
+        for fn_, mode in six.iteritems(files):
+            remote_path = _get_remote_path(fn_)
+
+            index = 1
+            failed = {}
+            for chunk in reader(fn_, chunk_size=self.opts['salt_cp_chunk_size']):
+                chunk = base64.b64encode(to_bytes(chunk))
+                append = index > 1
+                log.debug(
+                    'Copying %s to %starget \'%s\' as %s%s',
+                    fn_,
+                    '{0} '.format(selected_target_option)
+                        if selected_target_option
+                        else '',
+                    tgt,
+                    remote_path,
+                    ' (chunk #{0})'.format(index) if append else ''
+                )
+                args = [
+                    tgt,
+                    'cp.recv',
+                    [remote_path, chunk, append, gzip, mode],
+                    timeout,
                 ]
+                if selected_target_option is not None:
+                    args.append(selected_target_option)
 
-        selected_target_option = self.opts.get('selected_target_option', None)
+                result = local.cmd(*args)
-        if selected_target_option is not None:
-            args.append(selected_target_option)
 
-        ret = local.cmd(*args)
+                if not result:
+                    # Publish failed
+                    msg = (
+                        'Publish failed.{0} It may be necessary to '
+                        'decrease salt_cp_chunk_size (current value: '
+                        '{1})'.format(
+                            ' File partially transferred.' if index > 1 else '',
+                            self.opts['salt_cp_chunk_size'],
+                        )
+                    )
+                    for minion in minions:
+                        ret.setdefault(minion, {})[remote_path] = msg
+                    break
+
+                for minion_id, minion_ret in six.iteritems(result):
+                    ret.setdefault(minion_id, {})[remote_path] = minion_ret
+                    # Catch first error message for a given minion, we will
+                    # rewrite the results after we're done iterating through
+                    # the chunks.
+                    if minion_ret is not True and minion_id not in failed:
+                        failed[minion_id] = minion_ret
+
+                index += 1
+
+            for minion_id, msg in six.iteritems(failed):
+                ret[minion_id][remote_path] = msg
+
+        for dirname in empty_dirs:
+            remote_path = _get_remote_path(dirname)
+            log.debug(
+                'Creating empty dir %s on %starget \'%s\'',
+                dirname,
+                '{0} '.format(selected_target_option)
+                    if selected_target_option
+                    else '',
+                tgt,
+            )
+            args = [tgt, 'cp.recv', [remote_path, None], timeout]
+            if selected_target_option is not None:
+                args.append(selected_target_option)
+
+            for minion_id, minion_ret in six.iteritems(local.cmd(*args)):
+                ret.setdefault(minion_id, {})[remote_path] = minion_ret
 
         salt.output.display_output(
                 ret,

salt/client/__init__.py

@@ -1122,8 +1122,17 @@ class LocalClient(object):
                         minions.remove(raw['data']['id'])
                         break
                 except KeyError as exc:
-                    # This is a safe pass. We're just using the try/except to avoid having to deep-check for keys
+                    # This is a safe pass. We're just using the try/except to
-                    log.debug('Passing on saltutil error. This may be an error in saltclient. {0}'.format(exc))
+                    # avoid having to deep-check for keys.
+                    missing_key = exc.__str__().strip('\'"')
+                    if missing_key == 'retcode':
+                        log.debug('retcode missing from client return')
+                    else:
+                        log.debug(
+                            'Passing on saltutil error. Key \'%s\' missing '
+                            'from client return. This may be an error in '
+                            'the client.', missing_key
+                        )
                 # Keep track of the jid events to unsubscribe from later
                 open_jids.add(jinfo['jid'])
 

salt/config/__init__.py

@@ -958,6 +958,9 @@ VALID_OPTS = {
 
     # Permit or deny allowing minions to request revoke of its own key
     'allow_minion_key_revoke': bool,
+
+    # File chunk size for salt-cp
+    'salt_cp_chunk_size': int,
 }
 
 # default configurations
@@ -1201,6 +1204,7 @@ DEFAULT_MINION_OPTS = {
     'minion_jid_queue_hwm': 100,
     'ssl': None,
     'cache': 'localfs',
+    'salt_cp_chunk_size': 65536,
 }
 
 DEFAULT_MASTER_OPTS = {
@@ -1478,6 +1482,7 @@ DEFAULT_MASTER_OPTS = {
     'django_auth_path': '',
     'django_auth_settings': '',
     'allow_minion_key_revoke': True,
+    'salt_cp_chunk_size': 98304,
 }
 
 

salt/modules/cp.py

@@ -5,6 +5,8 @@ Minion side functions for salt-cp
 
 # Import python libs
 from __future__ import absolute_import
+import base64
+import errno
 import os
 import logging
 import fnmatch
@@ -13,6 +15,7 @@ import fnmatch
 import salt.minion
 import salt.fileclient
 import salt.utils
+import salt.utils.gzip_util
 import salt.utils.url
 import salt.crypt
 import salt.transport
@@ -54,33 +57,69 @@ def _gather_pillar(pillarenv, pillar_override):
     return ret
 
 
-def recv(files, dest):
+def recv(dest, chunk, append=False, compressed=True, mode=None):
     '''
-    Used with salt-cp, pass the files dict, and the destination.
+    This function receives files copied to the minion using ``salt-cp`` and is
-
+    not intended to be used directly on the CLI.
-    This function receives small fast copy files from the master via salt-cp.
-    It does not work via the CLI.
     '''
-    ret = {}
+    if 'retcode' not in __context__:
-    for path, data in six.iteritems(files):
+        __context__['retcode'] = 0
-        if os.path.basename(path) == os.path.basename(dest) \
-                and not os.path.isdir(dest):
-            final = dest
-        elif os.path.isdir(dest):
-            final = os.path.join(dest, os.path.basename(path))
-        elif os.path.isdir(os.path.dirname(dest)):
-            final = dest
-        else:
-            return 'Destination unavailable'
 
+    def _error(msg):
+        __context__['retcode'] = 1
+        return msg
+
+    if chunk is None:
+        # dest is an empty dir and needs to be created
         try:
-            with salt.utils.fopen(final, 'w+') as fp_:
+            os.makedirs(dest)
-                fp_.write(data)
+        except OSError as exc:
-            ret[final] = True
+            if exc.errno == errno.EEXIST:
-        except IOError:
+                if os.path.isfile(dest):
-            ret[final] = False
+                    return 'Path exists and is a file'
+            else:
+                return _error(exc.__str__())
+        return True
 
-    return ret
+    chunk = base64.b64decode(chunk)
+
+    open_mode = 'ab' if append else 'wb'
+    try:
+        fh_ = salt.utils.fopen(dest, open_mode)
+    except (IOError, OSError) as exc:
+        if exc.errno != errno.ENOENT:
+            # Parent dir does not exist, we need to create it
+            return _error(exc.__str__())
+        try:
+            os.makedirs(os.path.dirname(dest))
+        except (IOError, OSError) as makedirs_exc:
+            # Failed to make directory
+            return _error(makedirs_exc.__str__())
+        fh_ = salt.utils.fopen(dest, open_mode)
+
+    try:
+        # Write the chunk to disk
+        fh_.write(salt.utils.gzip_util.uncompress(chunk) if compressed
+                  else chunk)
+    except (IOError, OSError) as exc:
+        # Write failed
+        return _error(exc.__str__())
+    else:
+        # Write successful
+        if not append and mode is not None:
+            # If this is the first chunk we're writing, set the mode
+            #log.debug('Setting mode for %s to %s', dest, oct(mode))
+            log.debug('Setting mode for %s to %s', dest, mode)
+            try:
+                os.chmod(dest, mode)
+            except OSError:
+                return _error(exc.__str__())
+        return True
+    finally:
+        try:
+            fh_.close()
+        except AttributeError:
+            pass
 
 
 def _mk_client():

salt/modules/yumpkg.py

@@ -471,10 +471,11 @@ def latest_version(*names, **kwargs):
     def _check_cur(pkg):
         if pkg.name in cur_pkgs:
             for installed_version in cur_pkgs[pkg.name]:
-                # If any installed version is greater than the one found by
+                # If any installed version is greater than (or equal to) the
-                # yum/dnf list available, then it is not an upgrade.
+                # one found by yum/dnf list available, then it is not an
+                # upgrade.
                 if salt.utils.compare_versions(ver1=installed_version,
-                                               oper='>',
+                                               oper='>=',
                                                ver2=pkg.version,
                                                cmp_func=version_cmp):
                     return False

salt/utils/gzip_util.py

@@ -10,8 +10,11 @@ from __future__ import absolute_import
 # Import python libs
 import gzip
 
+# Import Salt libs
+import salt.utils
+
 # Import 3rd-party libs
-from salt.ext.six import BytesIO
+from salt.ext.six import BytesIO, StringIO
 
 
 class GzipFile(gzip.GzipFile):
@@ -63,3 +66,38 @@ def uncompress(data):
     with open_fileobj(buf, 'rb') as igz:
         unc = igz.read()
         return unc
+
+
+def compress_file(fh_, compresslevel=9, chunk_size=1048576):
+    '''
+    Generator that reads chunk_size bytes at a time from a file/filehandle and
+    yields the compressed result of each read.
+
+    .. note::
+        Each chunk is compressed separately. They cannot be stitched together
+        to form a compressed file. This function is designed to break up a file
+        into compressed chunks for transport and decompression/reassembly on a
+        remote host.
+    '''
+    try:
+        bytes_read = int(chunk_size)
+        if bytes_read != chunk_size:
+            raise ValueError
+    except ValueError:
+        raise ValueError('chunk_size must be an integer')
+    try:
+        while bytes_read == chunk_size:
+            buf = StringIO()
+            with open_fileobj(buf, 'wb', compresslevel) as ogz:
+                try:
+                    bytes_read = ogz.write(fh_.read(chunk_size))
+                except AttributeError:
+                    # Open the file and re-attempt the read
+                    fh_ = salt.utils.fopen(fh_, 'rb')
+                    bytes_read = ogz.write(fh_.read(chunk_size))
+            yield buf.getvalue()
+    finally:
+        try:
+            fh_.close()
+        except AttributeError:
+            pass

salt/utils/itertools.py

@@ -7,6 +7,9 @@ Helpful generators and other tools
 from __future__ import absolute_import
 import re
 
+# Import Salt libs
+import salt.utils
+
 
 def split(orig, sep=None):
     '''
@@ -32,3 +35,31 @@ def split(orig, sep=None):
         if pos < match.start() or sep is not None:
             yield orig[pos:match.start()]
         pos = match.end()
+
+
+def read_file(fh_, chunk_size=1048576):
+    '''
+    Generator that reads chunk_size bytes at a time from a file/filehandle and
+    yields it.
+    '''
+    try:
+        if chunk_size != int(chunk_size):
+            raise ValueError
+    except ValueError:
+        raise ValueError('chunk_size must be an integer')
+    try:
+        while True:
+            try:
+                chunk = fh_.read(chunk_size)
+            except AttributeError:
+                # Open the file and re-attempt the read
+                fh_ = salt.utils.fopen(fh_, 'rb')
+                chunk = fh_.read(chunk_size)
+            if not chunk:
+                break
+            yield chunk
+    finally:
+        try:
+            fh_.close()
+        except AttributeError:
+            pass

salt/utils/parsers.py

@@ -2108,6 +2108,17 @@ class SaltCPOptionParser(six.with_metaclass(OptionParserMeta,
     _default_logging_level_ = config.DEFAULT_MASTER_OPTS['log_level']
     _default_logging_logfile_ = config.DEFAULT_MASTER_OPTS['log_file']
 
+    def _mixin_setup(self):
+        file_opts_group = optparse.OptionGroup(self, 'File Options')
+        file_opts_group.add_option(
+            '-n', '--no-compression',
+            default=True,
+            dest='compression',
+            action='store_false',
+            help='Disable gzip compression.'
+        )
+        self.add_option_group(file_opts_group)
+
     def _mixin_after_parsed(self):
         # salt-cp needs arguments
         if len(self.args) <= 1:
@@ -2121,8 +2132,9 @@ class SaltCPOptionParser(six.with_metaclass(OptionParserMeta,
                 self.config['tgt'] = self.args[0].split()
         else:
             self.config['tgt'] = self.args[0]
-        self.config['src'] = self.args[1:-1]
+        self.config['src'] = [os.path.realpath(x) for x in self.args[1:-1]]
         self.config['dest'] = self.args[-1]
+        self.config['gzip'] = True
 
     def setup_config(self):
         return config.master_config(self.get_config_file_path())

tests/integration/shell/cp.py

@@ -9,10 +9,12 @@
 
 # Import python libs
 from __future__ import absolute_import
+import errno
 import os
 import yaml
 import pipes
 import shutil
+import tempfile
 
 # Import Salt Testing libs
 from salttesting.helpers import ensure_in_syspath
@@ -112,18 +114,13 @@ class CopyTest(integration.ShellCase, integration.ShellCaseCommonTestsMixIn):
             self.assertTrue(data[minion])
 
     def test_issue_7754(self):
-        try:
-            old_cwd = os.getcwd()
-        except OSError:
-            # Jenkins throws an OSError from os.getcwd()??? Let's not worry
-            # about it
-            old_cwd = None
-
         config_dir = os.path.join(integration.TMP, 'issue-7754')
-        if not os.path.isdir(config_dir):
-            os.makedirs(config_dir)
 
-        os.chdir(config_dir)
+        try:
+            os.makedirs(config_dir)
+        except OSError as exc:
+            if exc.errno != errno.EEXIST:
+                raise
 
         config_file_name = 'master'
         with salt.utils.fopen(self.get_config_file_path(config_file_name), 'r') as fhr:
@@ -134,15 +131,24 @@ class CopyTest(integration.ShellCase, integration.ShellCaseCommonTestsMixIn):
                     yaml.dump(config, default_flow_style=False)
                 )
 
-        ret = self.run_script(
-            self._call_binary_,
-            '--out pprint --config-dir {0} \'*\' foo {0}/foo'.format(
-                config_dir
-            ),
-            catch_stderr=True,
-            with_retcode=True
-        )
         try:
+            fd_, fn_ = tempfile.mkstemp()
+            os.close(fd_)
+
+            with salt.utils.fopen(fn_, 'w') as fp_:
+                fp_.write('Hello world!\n')
+
+            ret = self.run_script(
+                self._call_binary_,
+                '--out pprint --config-dir {0} \'*\' {1} {0}/{2}'.format(
+                    config_dir,
+                    fn_,
+                    os.path.basename(fn_),
+                ),
+                catch_stderr=True,
+                with_retcode=True
+            )
+
             self.assertIn('minion', '\n'.join(ret[0]))
             self.assertIn('sub_minion', '\n'.join(ret[0]))
             self.assertFalse(os.path.isdir(os.path.join(config_dir, 'file:')))
@@ -156,8 +162,11 @@ class CopyTest(integration.ShellCase, integration.ShellCaseCommonTestsMixIn):
             )
             self.assertEqual(ret[2], 2)
         finally:
-            if old_cwd is not None:
+            try:
-                self.chdir(old_cwd)
+                os.remove(fn_)
+            except OSError as exc:
+                if exc.errno != errno.ENOENT:
+                    raise
             if os.path.isdir(config_dir):
                 shutil.rmtree(config_dir)
 

tests/integration/states/pkg.py

@@ -33,7 +33,7 @@ __testcontext__ = {}
 _PKG_TARGETS = {
     'Arch': ['sl', 'libpng'],
     'Debian': ['python-plist', 'apg'],
-    'RedHat': ['xz-devel', 'zsh-html'],
+    'RedHat': ['units', 'zsh-html'],
     'FreeBSD': ['aalib', 'pth'],
     'Suse': ['aalib', 'python-pssh'],
     'MacOS': ['libpng', 'jpeg'],