Merge pull request #42890 from DSRCorporation/bugs/42627_salt-cp

Make chunked mode in salt-cp optional

doc/ref/cli/salt-cp.rst

@@ -39,6 +39,13 @@ specified target expression.
     desitination will be assumed to be a directory. Finally, recursion is now
     supported, allowing for entire directories to be copied.
 
+.. versionchanged:: 2016.11.7,2017.7.2
+    Reverted back to the old copy mode to preserve backward compatibility. The
+    new functionality added in 2016.6.6 and 2017.7.0 is now available using the
+    ``-C`` or ``--chunked`` CLI arguments. Note that compression, recursive
+    copying, and support for copying large files is only available in chunked
+    mode.
+
 Options
 =======
 
@@ -56,9 +63,16 @@ Options
 .. include:: _includes/target-selection.rst
 
 
+.. option:: -C, --chunked
+
+    Use new chunked mode to copy files. This mode supports large files, recursive
+    directories copying and compression.
+
+    .. versionadded:: 2016.11.7,2017.7.2
+
 .. option:: -n, --no-compression
 
-    Disable gzip compression.
+    Disable gzip compression in chunked mode.
 
     .. versionadded:: 2016.3.7,2016.11.6,Nitrogen
 

salt/cli/cp.py

@@ -19,8 +19,9 @@ import sys
 # Import salt libs
 import salt.client
 import salt.utils.gzip_util
+import salt.utils.itertools
 import salt.utils.minions
-from salt.utils import parsers, to_bytes
+from salt.utils import parsers, to_bytes, print_cli
 from salt.utils.verify import verify_log
 import salt.output
 
@@ -100,10 +101,69 @@ class SaltCP(object):
             empty_dirs.update(empty_dirs_)
         return files, sorted(empty_dirs)
 
+    def _file_dict(self, fn_):
+        '''
+        Take a path and return the contents of the file as a string
+        '''
+        if not os.path.isfile(fn_):
+            err = 'The referenced file, {0} is not available.'.format(fn_)
+            sys.stderr.write(err + '\n')
+            sys.exit(42)
+        with salt.utils.fopen(fn_, 'r') as fp_:
+            data = fp_.read()
+        return {fn_: data}
+
+    def _load_files(self):
+        '''
+        Parse the files indicated in opts['src'] and load them into a python
+        object for transport
+        '''
+        files = {}
+        for fn_ in self.opts['src']:
+            if os.path.isfile(fn_):
+                files.update(self._file_dict(fn_))
+            elif os.path.isdir(fn_):
+                print_cli(fn_ + ' is a directory, only files are supported in non-chunked mode. '
+                                'Use "--chunked" command line argument.')
+                sys.exit(1)
+        return files
+
     def run(self):
         '''
         Make the salt client call
         '''
+        if self.opts['chunked']:
+            ret = self.run_chunked()
+        else:
+            ret = self.run_oldstyle()
+
+        salt.output.display_output(
+                ret,
+                self.opts.get('output', 'nested'),
+                self.opts)
+
+    def run_oldstyle(self):
+        '''
+        Make the salt client call in old-style all-in-one call method
+        '''
+        arg = [self._load_files(), self.opts['dest']]
+        local = salt.client.get_local_client(self.opts['conf_file'])
+        args = [self.opts['tgt'],
+                'cp.recv',
+                arg,
+                self.opts['timeout'],
+                ]
+
+        selected_target_option = self.opts.get('selected_target_option', None)
+        if selected_target_option is not None:
+            args.append(selected_target_option)
+
+        return local.cmd(*args)
+
+    def run_chunked(self):
+        '''
+        Make the salt client call in the new fasion chunked multi-call way
+        '''
         files, empty_dirs = self._list_files()
         dest = self.opts['dest']
         gzip = self.opts['gzip']
@@ -165,7 +225,7 @@ class SaltCP(object):
                 )
                 args = [
                     tgt,
-                    'cp.recv',
+                    'cp.recv_chunked',
                     [remote_path, chunk, append, gzip, mode],
                     timeout,
                 ]
@@ -211,14 +271,11 @@ class SaltCP(object):
                     else '',
                 tgt,
             )
-            args = [tgt, 'cp.recv', [remote_path, None], timeout]
+            args = [tgt, 'cp.recv_chunked', [remote_path, None], timeout]
             if selected_target_option is not None:
                 args.append(selected_target_option)
 
             for minion_id, minion_ret in six.iteritems(local.cmd(*args)):
                 ret.setdefault(minion_id, {})[remote_path] = minion_ret
 
-        salt.output.display_output(
-                ret,
-                self.opts.get('output', 'nested'),
-                self.opts)
+        return ret

salt/modules/cp.py

@@ -57,7 +57,36 @@ def _gather_pillar(pillarenv, pillar_override):
     return ret
 
 
-def recv(dest, chunk, append=False, compressed=True, mode=None):
+def recv(files, dest):
+    '''
+    Used with salt-cp, pass the files dict, and the destination.
+
+    This function receives small fast copy files from the master via salt-cp.
+    It does not work via the CLI.
+    '''
+    ret = {}
+    for path, data in six.iteritems(files):
+        if os.path.basename(path) == os.path.basename(dest) \
+                and not os.path.isdir(dest):
+            final = dest
+        elif os.path.isdir(dest):
+            final = os.path.join(dest, os.path.basename(path))
+        elif os.path.isdir(os.path.dirname(dest)):
+            final = dest
+        else:
+            return 'Destination unavailable'
+
+        try:
+            with salt.utils.fopen(final, 'w+') as fp_:
+                fp_.write(data)
+            ret[final] = True
+        except IOError:
+            ret[final] = False
+
+    return ret
+
+
+def recv_chunked(dest, chunk, append=False, compressed=True, mode=None):
     '''
     This function receives files copied to the minion using ``salt-cp`` and is
     not intended to be used directly on the CLI.

salt/utils/parsers.py

@@ -2110,10 +2110,18 @@ class SaltCPOptionParser(six.with_metaclass(OptionParserMeta,
 
     def _mixin_setup(self):
         file_opts_group = optparse.OptionGroup(self, 'File Options')
+        file_opts_group.add_option(
+            '-C', '--chunked',
+            default=False,
+            dest='chunked',
+            action='store_true',
+            help='Use chunked files transfer. Supports big files, recursive '
+                 'lookup and directories creation.'
+        )
         file_opts_group.add_option(
             '-n', '--no-compression',
             default=True,
-            dest='compression',
+            dest='gzip',
             action='store_false',
             help='Disable gzip compression.'
         )
@@ -2134,7 +2142,6 @@ class SaltCPOptionParser(six.with_metaclass(OptionParserMeta,
             self.config['tgt'] = self.args[0]
         self.config['src'] = [os.path.realpath(x) for x in self.args[1:-1]]
         self.config['dest'] = self.args[-1]
-        self.config['gzip'] = True
 
     def setup_config(self):
         return config.master_config(self.get_config_file_path())

tests/integration/files/file/base/_modules/runtests_helpers.py

@@ -50,6 +50,7 @@ def get_invalid_docs():
     allow_failure = (
         'cmd.win_runas',
         'cp.recv',
+        'cp.recv_chunked',
         'glance.warn_until',
         'ipset.long_range',
         'libcloud_dns.get_driver',