Merge pull request #26212 from DSRCompany/http_client_issues

Fix for Tornado HTTPClient file size and timeout limits issues.

salt/fileclient.py

@@ -593,35 +593,38 @@ class Client(object):
             get_kwargs['auth'] = (url_data.username, url_data.password)
         else:
             fixed_url = url
+
+        destfp = None
         try:
+            if no_cache:
+                result = []
+
+                def on_chunk(chunk):
+                    result.append(chunk)
+            else:
+                dest_tmp = "{0}.part".format(dest)
+                destfp = salt.utils.fopen(dest_tmp, 'wb')
+
+                def on_chunk(chunk):
+                    destfp.write(chunk)
+
             query = salt.utils.http.query(
                 fixed_url,
                 stream=True,
+                streaming_callback=on_chunk,
                 username=url_data.username,
                 password=url_data.password,
                 **get_kwargs
             )
             if 'handle' not in query:
                 raise MinionError('Error: {0}'.format(query['error']))
-            response = query['handle']
-            chunk_size = 32 * 1024
-            if not no_cache:
-                with salt.utils.fopen(dest, 'wb') as destfp:
-                    if hasattr(response, 'iter_content'):
-                        for chunk in response.iter_content(chunk_size=chunk_size):
-                            destfp.write(chunk)
-                    else:
-                        while True:
-                            chunk = response.buffer.read(chunk_size)
-                            destfp.write(chunk)
-                            if len(chunk) < chunk_size:
-                                break
-                return dest
+            if no_cache:
+                return ''.join(result)
             else:
-                if hasattr(response, 'text'):
-                    return response.text
-                else:
-                    return response['text']
+                destfp.close()
+                destfp = None
+                os.rename(dest_tmp, dest)
+                return dest
         except HTTPError as exc:
             raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                 exc.code,
@@ -629,6 +632,9 @@ class Client(object):
                 *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
         except URLError as exc:
             raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
+        finally:
+            if destfp is not None:
+                destfp.close()
 
     def get_template(
             self,

salt/utils/http.py

@@ -118,6 +118,7 @@ def query(url,
           headers_out=None,
           decode_out=None,
           stream=False,
+          streaming_callback=None,
           handle=False,
           agent=USERAGENT,
           hide_fields=None,
@@ -403,7 +404,7 @@ def query(url,
             data = urllib.urlencode(data)
 
         try:
-            result = HTTPClient().fetch(
+            result = HTTPClient(max_body_size=100*1024*1024*1024).fetch(
                 url_full,
                 method=method,
                 headers=header_dict,
@@ -412,6 +413,8 @@ def query(url,
                 body=data,
                 validate_cert=verify_ssl,
                 allow_nonstandard_methods=True,
+                streaming_callback=streaming_callback,
+                request_timeout=3600.0,
                 **req_kwargs
             )
         except tornado.httpclient.HTTPError as exc: